diff --git a/projects/hipcub/.clang-format b/projects/hipcub/.clang-format index ddb455408800..21f7c2598084 100644 --- a/projects/hipcub/.clang-format +++ b/projects/hipcub/.clang-format @@ -167,10 +167,4 @@ Macros: - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS=[[DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS___]] BreakAfterAttributes: Always -WhitespaceSensitiveMacros: [ - 'HIPCUB_HAS_INCLUDE', - '_HIPCUB_LIBCXX_INCLUDE', - '_HIPCUB_STD_INCLUDE' -] - --- diff --git a/projects/hipcub/CHANGELOG.md b/projects/hipcub/CHANGELOG.md index ced06d436796..30b45edaf426 100644 --- a/projects/hipcub/CHANGELOG.md +++ b/projects/hipcub/CHANGELOG.md @@ -2,33 +2,6 @@ Full documentation for hipCUB is available at [https://rocm.docs.amd.com/projects/hipCUB/en/latest/](https://rocm.docs.amd.com/projects/hipCUB/en/latest/). -## hipCUB-5.0.0 for ROCm 10.0 - -### Added - -* Added `::hip::std` support. - -### Changed - -* Changed `CCCL_MINIMUM_VERSION` to `3.0.0` to align with CUB. -* Add support for large num_items `DeviceMerge` and `DeviceSegmentedSort`. -* Replace `#pragma unroll` by `_CCCL_PRAGMA_UNROLL_FULL()` and `_CCCL_PRAGMA_NOUNROLL()` by `_CCCL_PRAGMA_NOUNROLL()`. -* Add `_CCCL_SORT_MAYBE_UNROLL()` in block merge sort and thread sort. -* Update `WarpExchange` template parameters for CUB compatibility. - -### Removed - -* Removed `hipcub::BaseTraits::CATEGORY`, `hipcub::BaseTraits::nullptr_TYPE` and `hipcub::BaseTraits::PRIMITIVE`. -* Removed `ConstantInputIterator`, `CountingInputIterator`, `DiscardOutputIterator` and `TransformInputIterator` which were deprecated in hipCUB-4.1.0. -* Removed `DeviceSpmv`, which was removed from CUB after CCCL's 2.8.0 release. Use `hipSPARSE` or `rocSPARSE` libraries instead. -* Removed `GridBarrier`. -* Removed `HIPCUB_MIN`, `HIPCUB_MAX`, `HIPCUB_QUOTIENT_FLOOR`, `HIPCUB_QUOTIENT_CEILING`, `HIPCUB_ROUND_UP_NEAREST` and `HIPCUB_ROUND_DOWN_NEAREST` which were deprecated in hipCUB-4.1.0. -* Removed `LEGACY_PTX_ARCH`. -* Removed `hipcub:max` and `hipcub:min`, which were deprecated. Use `hip::std::max` and `hip::std::min` instead. -* Deprecated `hipcub::Swap`, use `rocprim::swap` instead. -* Deprecated `HIPCUB_IS_INT128_ENABLED`, use `_CCCL_HAS_INT128()` instead. -* Deprecated `hipcub::Equality`, `hipcub::Inequality`, `hipcub::InequalityWrapper`, `hipcub::Sum`, `hipcub::Difference`, `hipcub::Division`, `hipcub::Max` and `hipcub::Min` operators. Use `hip::std::equal_to`, `hip::std::not_equal_to`, `hip::std::plus`, `hip::std::minus`, `hip::std::divides`, `hip::maximum` and `hip:minimum` operators instead. - ## hipCUB 4.5.0 for ROCm 7.14 ### Added diff --git a/projects/hipcub/CMakeLists.txt b/projects/hipcub/CMakeLists.txt index 598d8175ead0..2772f4f9ab8d 100644 --- a/projects/hipcub/CMakeLists.txt +++ b/projects/hipcub/CMakeLists.txt @@ -35,11 +35,11 @@ endif() # Update these variables at release time # # Set the library version -set(VERSION_STRING "5.0.0") +set(VERSION_STRING "4.5.0") # Set the CCCL-compatible version. -set(HIPCUB_CCCL_VERSION_MAJOR 3) -set(HIPCUB_CCCL_VERSION_MINOR 0) -set(HIPCUB_CCCL_VERSION_PATCH 3) +set(HIPCUB_CCCL_VERSION_MAJOR 2) +set(HIPCUB_CCCL_VERSION_MINOR 8) +set(HIPCUB_CCCL_VERSION_PATCH 2) # Set the minimum required rocPRIM version set(MIN_ROCPRIM_PACKAGE_VERSION "4.1.0" CACHE STRING "Minimum version of rocPRIM to search for when ROCPRIM_FETCH_METHOD is set to PACKAGE.") # Set download branch for dependency rocPRIM @@ -59,8 +59,10 @@ endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -if(NOT CMAKE_CXX_STANDARD EQUAL 17) - message(FATAL_ERROR "Only C++17 are supported") +if (CMAKE_CXX_STANDARD EQUAL 14) + message(WARNING "C++14 will be deprecated in the next major release") +elseif(NOT CMAKE_CXX_STANDARD EQUAL 17) + message(FATAL_ERROR "Only C++14 and C++17 are supported") endif() # Set HIP flags diff --git a/projects/hipcub/benchmark/CMakeLists.txt b/projects/hipcub/benchmark/CMakeLists.txt index 9679f1285f07..3e8f663d1a1b 100644 --- a/projects/hipcub/benchmark/CMakeLists.txt +++ b/projects/hipcub/benchmark/CMakeLists.txt @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,11 +35,6 @@ function(add_hipcub_benchmark BENCHMARK_SOURCE) benchmark::benchmark hipcub ) - - if (WIN32) - target_compile_definitions(${BENCHMARK_TARGET} PRIVATE BENCHMARK_STATIC_DEFINE) - endif() - if((HIP_COMPILER STREQUAL "nvcc")) set_property(TARGET ${BENCHMARK_TARGET} PROPERTY CUDA_STANDARD 17) set_source_files_properties(${BENCHMARK_SOURCE} PROPERTIES LANGUAGE CUDA) @@ -105,6 +100,7 @@ add_hipcub_benchmark(benchmark_device_segmented_sort.cpp) add_hipcub_benchmark(benchmark_device_segmented_radix_sort.cpp) add_hipcub_benchmark(benchmark_device_segmented_reduce.cpp) add_hipcub_benchmark(benchmark_device_select.cpp) +add_hipcub_benchmark(benchmark_device_spmv.cpp) add_hipcub_benchmark(benchmark_warp_exchange.cpp) add_hipcub_benchmark(benchmark_warp_load.cpp) add_hipcub_benchmark(benchmark_warp_reduce.cpp) diff --git a/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp b/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp index 4581111bb936..7c7ac6bc1938 100644 --- a/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp +++ b/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2022 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -63,7 +63,7 @@ struct subtract_left hipcub::BlockAdjacentDifference adjacent_difference; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; @@ -106,7 +106,7 @@ struct subtract_left_partial_tile // Try to evenly distribute the length of tile_sizes between all the trials const auto tile_size_diff = (BlockSize * ItemsPerThread) / trials + 1; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; @@ -150,7 +150,7 @@ struct subtract_right hipcub::BlockAdjacentDifference adjacent_difference; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; @@ -193,7 +193,7 @@ struct subtract_right_partial_tile // Try to evenly distribute the length of tile_sizes between all the trials const auto tile_size_diff = (BlockSize * ItemsPerThread) / trials + 1; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; @@ -221,8 +221,8 @@ template auto run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) - -> std::enable_if_t - && !std::is_same_v> + -> std::enable_if_t::value + && !std::is_same::value> { constexpr auto items_per_block = BlockSize * ItemsPerThread; const auto num_blocks = (N + items_per_block - 1) / items_per_block; @@ -271,8 +271,8 @@ template auto run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) - -> std::enable_if_t - || std::is_same_v> + -> std::enable_if_t::value + || std::is_same::value> { constexpr auto items_per_block = BlockSize * ItemsPerThread; const auto num_blocks = (N + items_per_block - 1) / items_per_block; @@ -352,7 +352,7 @@ void add_benchmarks(const std::string& name, BENCHMARK_TYPE(long long, 256, false), BENCHMARK_TYPE(double, 256, false)}; - if(!std::is_same_v) + if(!std::is_same::value) { bs.insert(bs.end(), {BENCHMARK_TYPE(int, 256, true), diff --git a/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp b/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp index 80666bcfbaaa..5e36160c1405 100644 --- a/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp +++ b/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ #include #include #include +#include //to use hipcub::Equality #include "common_benchmark_header.hpp" @@ -31,13 +32,13 @@ const size_t DEFAULT_N = 1024 * 1024 * 128; #endif -struct equal +template +struct custom_flag_op1 { - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const A& a, const B& b) const + HIPCUB_HOST_DEVICE + bool operator()(const T& a, const T& b) const { - return a == b; + return (a == b); } }; @@ -67,17 +68,17 @@ struct flag_heads T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockDiscontinuity bdiscontinuity; bool head_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.FlagHeads(head_flags, input, equal(), T(123)); + bdiscontinuity.FlagHeads(head_flags, input, hipcub::Equality(), T(123)); } else { - bdiscontinuity.FlagHeads(head_flags, input, equal()); + bdiscontinuity.FlagHeads(head_flags, input, hipcub::Equality()); } for(unsigned int i = 0; i < ItemsPerThread; i++) @@ -105,17 +106,17 @@ struct flag_tails T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockDiscontinuity bdiscontinuity; bool tail_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.FlagTails(tail_flags, input, equal(), T(123)); + bdiscontinuity.FlagTails(tail_flags, input, hipcub::Equality(), T(123)); } else { - bdiscontinuity.FlagTails(tail_flags, input, equal()); + bdiscontinuity.FlagTails(tail_flags, input, hipcub::Equality()); } for(unsigned int i = 0; i < ItemsPerThread; i++) @@ -143,7 +144,7 @@ struct flag_heads_and_tails T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockDiscontinuity bdiscontinuity; @@ -151,12 +152,15 @@ struct flag_heads_and_tails bool tail_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity - .FlagHeadsAndTails(head_flags, T(123), tail_flags, T(234), input, equal()); - } - else + bdiscontinuity.FlagHeadsAndTails(head_flags, + T(123), + tail_flags, + T(234), + input, + hipcub::Equality()); + } else { - bdiscontinuity.FlagHeadsAndTails(head_flags, tail_flags, input, equal()); + bdiscontinuity.FlagHeadsAndTails(head_flags, tail_flags, input, hipcub::Equality()); } for(unsigned int i = 0; i < ItemsPerThread; i++) diff --git a/projects/hipcub/benchmark/benchmark_block_exchange.cpp b/projects/hipcub/benchmark/benchmark_block_exchange.cpp index 602c77bc523f..000cd41be6a0 100644 --- a/projects/hipcub/benchmark/benchmark_block_exchange.cpp +++ b/projects/hipcub/benchmark/benchmark_block_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -54,7 +54,7 @@ struct blocked_to_striped T input[ItemsPerThread]; hipcub::LoadDirectBlocked(lid, d_input + block_offset, input); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -78,7 +78,7 @@ struct striped_to_blocked T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -102,7 +102,7 @@ struct blocked_to_warp_striped T input[ItemsPerThread]; hipcub::LoadDirectBlocked(lid, d_input + block_offset, input); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -126,7 +126,7 @@ struct warp_striped_to_blocked T input[ItemsPerThread]; hipcub::LoadDirectWarpStriped(lid, d_input + block_offset, input); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -152,7 +152,7 @@ struct scatter_to_blocked hipcub::LoadDirectStriped(lid, d_input + block_offset, input); hipcub::LoadDirectStriped(lid, d_ranks + block_offset, ranks); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -178,7 +178,7 @@ struct scatter_to_striped hipcub::LoadDirectStriped(lid, d_input + block_offset, input); hipcub::LoadDirectStriped(lid, d_ranks + block_offset, ranks); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; diff --git a/projects/hipcub/benchmark/benchmark_block_histogram.cpp b/projects/hipcub/benchmark/benchmark_block_histogram.cpp index e593023d6bfd..1206e0423537 100644 --- a/projects/hipcub/benchmark/benchmark_block_histogram.cpp +++ b/projects/hipcub/benchmark/benchmark_block_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -64,13 +64,13 @@ struct histogram __shared__ T histogram[BinSize]; __shared__ typename bhistogram_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { bhistogram_t(storage).Histogram(values, histogram); } - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int offset = 0; offset < BinSize; offset += BlockSize) { if(offset + hipThreadIdx_x < BinSize) diff --git a/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp b/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp index 42221bc66079..c8c7402b1366 100644 --- a/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -52,7 +52,7 @@ void sort_keys_kernel(const T* input, T* output, CompareOp compare_op) T keys[ItemsPerThread]; hipcub::LoadDirectStriped(lid, input + block_offset, keys); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockMergeSort sort; @@ -82,7 +82,7 @@ void sort_pairs_kernel(const T* input, T* output, CompareOp compare_op) values[i] = keys[i] + T(1); } - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockMergeSort sort; diff --git a/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp b/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp index 0f12ebd20fae..ffcd1d775055 100644 --- a/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp +++ b/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -70,7 +70,7 @@ __global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_input, in Descending, BenchmarkKind == RadixRankAlgorithm::RADIX_RANK_MEMOIZE>>; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int key = 0; key < ItemsPerThread; key++) { unsigned_keys[key] = KeyTraits::TwiddleIn(unsigned_keys[key]); @@ -78,7 +78,7 @@ __global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_input, in int ranks[ItemsPerThread]; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { __shared__ typename RankType::TempStorage storage; diff --git a/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp b/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp index ca2a0809391a..4b75c26910a4 100644 --- a/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -134,7 +134,7 @@ __global__ __launch_bounds__(BlockSize) void sort_keys_kernel(const T* input, T* T keys[ItemsPerThread]; Helper::template load(lid, input + block_offset, keys); - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { Helper::template sort(keys); @@ -162,7 +162,7 @@ __global__ __launch_bounds__(BlockSize) void sort_pairs_kernel(const T* input, T values[i] = keys[i] + T(1); } - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { Helper::template sort(keys, values); diff --git a/projects/hipcub/benchmark/benchmark_block_reduce.cpp b/projects/hipcub/benchmark/benchmark_block_reduce.cpp index 53b27b32c327..fe4b815d50c1 100644 --- a/projects/hipcub/benchmark/benchmark_block_reduce.cpp +++ b/projects/hipcub/benchmark/benchmark_block_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ // HIP API #include +#include #ifndef DEFAULT_N const size_t DEFAULT_N = 1024 * 1024 * 32; @@ -57,10 +58,10 @@ struct reduce using breduce_t = hipcub::BlockReduce; __shared__ typename breduce_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { - reduced_value = breduce_t(storage).Reduce(values, benchmark_utils::plus{}); + reduced_value = breduce_t(storage).Reduce(values, hipcub::Sum()); values[0] = reduced_value; } diff --git a/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp b/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp index ab317604c300..a42d3c480651 100644 --- a/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp +++ b/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -58,7 +58,7 @@ __global__ = d_run_offsets[(hipBlockIdx_x + 1) * BlockSize * RunsPerThread] - d_run_offsets[hipBlockIdx_x * BlockSize * RunsPerThread]; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned i = 0; i < Trials; ++i) { OffsetT decoded_window_offset = 0; diff --git a/projects/hipcub/benchmark/benchmark_block_scan.cpp b/projects/hipcub/benchmark/benchmark_block_scan.cpp index 7732e61c0fd5..51bf6c63fac7 100644 --- a/projects/hipcub/benchmark/benchmark_block_scan.cpp +++ b/projects/hipcub/benchmark/benchmark_block_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2022 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -57,10 +57,10 @@ struct inclusive_scan using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { - bscan_t(storage).InclusiveScan(values, values, benchmark_utils::plus{}); + bscan_t(storage).InclusiveScan(values, values, hipcub::Sum()); } for(unsigned int k = 0; k < ItemsPerThread; k++) @@ -87,10 +87,10 @@ struct exclusive_scan using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { - bscan_t(storage).ExclusiveScan(values, values, init, benchmark_utils::plus{}); + bscan_t(storage).ExclusiveScan(values, values, init, hipcub::Sum()); } for(unsigned int k = 0; k < ItemsPerThread; k++) diff --git a/projects/hipcub/benchmark/benchmark_block_shuffle.cpp b/projects/hipcub/benchmark/benchmark_block_shuffle.cpp index 6a958198f0f8..697d381c24dc 100644 --- a/projects/hipcub/benchmark/benchmark_block_shuffle.cpp +++ b/projects/hipcub/benchmark/benchmark_block_shuffle.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -53,7 +53,7 @@ struct offset using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Offset(value, value, 1); @@ -84,7 +84,7 @@ struct rotate using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Rotate(value, value, 1); @@ -116,7 +116,7 @@ struct up using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Up(values, values); @@ -151,7 +151,7 @@ struct down using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Down(values, values); diff --git a/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp b/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp index d3d57da67d12..335144c0248e 100644 --- a/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp +++ b/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -43,7 +43,7 @@ namespace { #ifndef DEFAULT_N -constexpr size_t DEFAULT_N = 1024 * 1024 * 128; +constexpr std::size_t DEFAULT_N = 1024 * 1024 * 128; #endif constexpr unsigned int batch_size = 10; @@ -53,7 +53,7 @@ template auto dispatch_adjacent_difference(std::true_type /*left*/, std::true_type /*copy*/, void* const temporary_storage, - size_t& storage_size, + std::size_t& storage_size, const InputIt input, const OutputIt output, Args&&... args) @@ -69,7 +69,7 @@ template auto dispatch_adjacent_difference(std::false_type /*left*/, std::true_type /*copy*/, void* const temporary_storage, - size_t& storage_size, + std::size_t& storage_size, const InputIt input, const OutputIt output, Args&&... args) @@ -85,7 +85,7 @@ template auto dispatch_adjacent_difference(std::true_type /*left*/, std::false_type /*copy*/, void* const temporary_storage, - size_t& storage_size, + std::size_t& storage_size, const InputIt input, const OutputIt /*output*/, Args&&... args) @@ -100,7 +100,7 @@ template auto dispatch_adjacent_difference(std::false_type /*left*/, std::false_type /*copy*/, void* const temporary_storage, - size_t& storage_size, + std::size_t& storage_size, const InputIt input, const OutputIt /*output*/, Args&&... args) @@ -112,7 +112,7 @@ auto dispatch_adjacent_difference(std::false_type /*left*/, } template -void run_benchmark(benchmark::State& state, const size_t size, const hipStream_t stream) +void run_benchmark(benchmark::State& state, const std::size_t size, const hipStream_t stream) { using output_type = T; @@ -134,7 +134,7 @@ void run_benchmark(benchmark::State& state, const size_t size, const hipStream_t static constexpr std::integral_constant copy_tag; // Allocate temporary storage - size_t temp_storage_size{}; + std::size_t temp_storage_size{}; void* d_temp_storage = nullptr; const auto launch = [&] @@ -146,7 +146,7 @@ void run_benchmark(benchmark::State& state, const size_t size, const hipStream_t d_input, d_output, size, - benchmark_utils::plus{}, + hipcub::Sum{}, stream); }; HIP_CHECK(launch()); @@ -237,7 +237,7 @@ int main(int argc, char* argv[]) // Add benchmarks const std::vector benchmarks = { CREATE_BENCHMARKS(int), - CREATE_BENCHMARKS(_HIPCUB_STD::int64_t), + CREATE_BENCHMARKS(std::int64_t), CREATE_BENCHMARKS(uint8_t), diff --git a/projects/hipcub/benchmark/benchmark_device_histogram.cpp b/projects/hipcub/benchmark/benchmark_device_histogram.cpp index d8ff79734b28..ded31e28f8cb 100644 --- a/projects/hipcub/benchmark/benchmark_device_histogram.cpp +++ b/projects/hipcub/benchmark/benchmark_device_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -30,6 +30,7 @@ // HIP API #include +#include #ifndef DEFAULT_N const size_t DEFAULT_N = 1024 * 1024 * 32; @@ -53,7 +54,7 @@ std::vector std::default_random_engine gen(rd()); std::vector data(size); std::generate(data.begin(), - data.begin() + _HIPCUB_STD::min(size, max_random_size), + data.begin() + std::min(size, max_random_size), [&]() { // Reduce entropy by applying bitwise AND to random bits @@ -68,7 +69,7 @@ std::vector }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(data.begin(), _HIPCUB_STD::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); } return data; } @@ -463,7 +464,7 @@ struct num_limits { static constexpr T max() { - return _HIPCUB_STD::numeric_limits::max(); + return std::numeric_limits::max(); }; }; diff --git a/projects/hipcub/benchmark/benchmark_device_memory.cpp b/projects/hipcub/benchmark/benchmark_device_memory.cpp index d00007f4e327..1e62167a7a1a 100644 --- a/projects/hipcub/benchmark/benchmark_device_memory.cpp +++ b/projects/hipcub/benchmark/benchmark_device_memory.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -81,12 +81,12 @@ struct operation (void)storage; (void)global_mem_output; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int i = 0; i < ItemsPerThread; i++) { input[i] = input[i] + 666; constexpr unsigned int repeats = 30; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int j = 0; j < repeats; j++) { input[i] = input[i] * (input[j % ItemsPerThread]); @@ -113,7 +113,7 @@ struct operation // sync before re-using shared memory from load __syncthreads(); - block_scan_type(storage).InclusiveScan(input, input, benchmark_utils::plus{}); + block_scan_type(storage).InclusiveScan(input, input, hipcub::Sum()); } }; @@ -134,7 +134,7 @@ struct operation const unsigned int index = threadIdx.x * ItemsPerThread + blockIdx.x * blockDim.x * ItemsPerThread; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int i = 0; i < ItemsPerThread; i++) { atomicAdd(&global_mem_output[index + i], T(666)); @@ -159,7 +159,7 @@ struct operation const unsigned int index = (threadIdx.x % warpSize) * ItemsPerThread + blockIdx.x * blockDim.x * ItemsPerThread; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int i = 0; i < ItemsPerThread; i++) { atomicAdd(&global_mem_output[index + i], T(666)); @@ -183,7 +183,7 @@ struct operation (void)input; const unsigned int index = threadIdx.x * ItemsPerThread; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int i = 0; i < ItemsPerThread; i++) { atomicAdd(&global_mem_output[index + i], T(666)); diff --git a/projects/hipcub/benchmark/benchmark_device_reduce.cpp b/projects/hipcub/benchmark/benchmark_device_reduce.cpp index 22f38c7d0e6d..8dcd96861a96 100644 --- a/projects/hipcub/benchmark/benchmark_device_reduce.cpp +++ b/projects/hipcub/benchmark/benchmark_device_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -89,7 +89,7 @@ template struct Benchmark; template -struct Benchmark +struct Benchmark { static void run(benchmark::State& state, size_t size, const hipStream_t stream) { @@ -100,7 +100,7 @@ struct Benchmark }; template -struct Benchmark +struct Benchmark { static void run(benchmark::State& state, size_t size, const hipStream_t stream) { @@ -166,11 +166,11 @@ int main(int argc, char* argv[]) // Add benchmarks std::vector benchmarks = { - CREATE_BENCHMARKS(benchmark_utils::plus), - CREATE_BENCHMARK(custom_double2, benchmark_utils::plus), - CREATE_BENCHMARKS(benchmark_utils::minimum), + CREATE_BENCHMARKS(hipcub::Sum), + CREATE_BENCHMARK(custom_double2, hipcub::Sum), + CREATE_BENCHMARKS(hipcub::Min), #ifdef HIPCUB_ROCPRIM_API - CREATE_BENCHMARK(custom_double2, benchmark_utils::minimum), + CREATE_BENCHMARK(custom_double2, hipcub::Min), #endif CREATE_BENCHMARKS(hipcub::ArgMin), #ifdef HIPCUB_ROCPRIM_API diff --git a/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp b/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp index d57d6249392e..0d9160f4c9c7 100644 --- a/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp +++ b/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -58,7 +58,7 @@ void run_benchmark(benchmark::State& state, while(offset < size) { const size_t key_count = key_counts[unique_count % key_counts.size()]; - const size_t end = _HIPCUB_STD::min(size, offset + key_count); + const size_t end = std::min(size, offset + key_count); for(size_t i = offset; i < end; i++) { keys_input[i] = unique_count; @@ -187,11 +187,11 @@ void add_benchmarks(size_t max_length, using custom_double2 = benchmark_utils::custom_type; std::vector bs = { - CREATE_BENCHMARKS(benchmark_utils::plus), - CREATE_BENCHMARK(long long, custom_double2, benchmark_utils::plus), - CREATE_BENCHMARKS(benchmark_utils::minimum), + CREATE_BENCHMARKS(hipcub::Sum), + CREATE_BENCHMARK(long long, custom_double2, hipcub::Sum), + CREATE_BENCHMARKS(hipcub::Min), #ifdef HIPCUB_ROCPRIM_API - CREATE_BENCHMARK(long long, custom_double2, benchmark_utils::minimum), + CREATE_BENCHMARK(long long, custom_double2, hipcub::Min), #endif }; diff --git a/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp b/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp index e3e24e56ccaa..f0c858528a53 100644 --- a/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp +++ b/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -54,7 +54,7 @@ void run_encode_benchmark(benchmark::State& state, while(offset < size) { const size_t key_count = key_counts[runs_count % key_counts.size()]; - const size_t end = _HIPCUB_STD::min(size, offset + key_count); + const size_t end = std::min(size, offset + key_count); for(size_t i = offset; i < end; i++) { input[i] = runs_count; @@ -157,7 +157,7 @@ void run_non_trivial_runs_benchmark(benchmark::State& state, while(offset < size) { const size_t key_count = key_counts[runs_count % key_counts.size()]; - const size_t end = _HIPCUB_STD::min(size, offset + key_count); + const size_t end = std::min(size, offset + key_count); for(size_t i = offset; i < end; i++) { input[i] = runs_count; diff --git a/projects/hipcub/benchmark/benchmark_device_scan.cpp b/projects/hipcub/benchmark/benchmark_device_scan.cpp index 54706c2ae4b5..5d38b9628b5c 100644 --- a/projects/hipcub/benchmark/benchmark_device_scan.cpp +++ b/projects/hipcub/benchmark/benchmark_device_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -29,11 +29,8 @@ #include "common_benchmark_header.hpp" // HIP API -#include #include -#include _HIPCUB_STD_INCLUDE(functional) - #ifndef DEFAULT_N const size_t DEFAULT_N = 1024 * 1024 * 32; #endif @@ -100,7 +97,7 @@ auto run_device_scan_by_key(void* temporary_storage, scan_op, initial_value, static_cast(input_size), - _HIPCUB_STD::equal_to<>(), + hipcub::Equality(), stream); } @@ -123,7 +120,7 @@ auto run_device_scan_by_key(void* temporary_storage, output, scan_op, static_cast(input_size), - _HIPCUB_STD::equal_to<>(), + hipcub::Equality(), stream); } @@ -352,8 +349,8 @@ int main(int argc, char* argv[]) // Add benchmarks std::vector benchmarks = { - CREATE_BENCHMARKS(benchmark_utils::plus), - CREATE_BENCHMARKS(benchmark_utils::minimum), + CREATE_BENCHMARKS(hipcub::Sum), + CREATE_BENCHMARKS(hipcub::Min), }; // Use manual timing diff --git a/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp b/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp index 7159de1e4869..5ba4a94284fb 100644 --- a/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp +++ b/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp @@ -147,38 +147,24 @@ template struct Benchmark; template -struct Benchmark +struct Benchmark { static void run(benchmark::State& state, size_t desired_segments, const hipStream_t stream, size_t size) { - hipError_t (*ptr_to_sum)(void*, - size_t&, - T*, - T*, - _HIPCUB_STD::int64_t, - OffsetType*, - OffsetType*, - hipStream_t) + hipError_t (*ptr_to_sum)(void*, size_t&, T*, T*, int, OffsetType*, OffsetType*, hipStream_t) = &hipcub::DeviceSegmentedReduce::Sum; run_benchmark(state, desired_segments, stream, size, ptr_to_sum); } }; template -struct Benchmark +struct Benchmark { static void run(benchmark::State& state, size_t desired_segments, const hipStream_t stream, size_t size) { - hipError_t (*ptr_to_min)(void*, - size_t&, - T*, - T*, - _HIPCUB_STD::int64_t, - OffsetType*, - OffsetType*, - hipStream_t) + hipError_t (*ptr_to_min)(void*, size_t&, T*, T*, int, OffsetType*, OffsetType*, hipStream_t) = &hipcub::DeviceSegmentedReduce::Min; run_benchmark(state, desired_segments, stream, size, ptr_to_min); } @@ -198,7 +184,7 @@ struct Benchmark size_t&, T*, KeyValue*, - _HIPCUB_STD::int64_t, + int, OffsetType*, OffsetType*, hipStream_t) @@ -233,11 +219,11 @@ void add_benchmarks(std::vector& benchmarks, using custom_double2 = benchmark_utils::custom_type; std::vector bs = { - CREATE_BENCHMARKS(benchmark_utils::plus), - BENCHMARK_TYPE(custom_double2, benchmark_utils::plus), - CREATE_BENCHMARKS(benchmark_utils::minimum), + CREATE_BENCHMARKS(hipcub::Sum), + BENCHMARK_TYPE(custom_double2, hipcub::Sum), + CREATE_BENCHMARKS(hipcub::Min), #ifdef HIPCUB_ROCPRIM_API - BENCHMARK_TYPE(custom_double2, benchmark_utils::minimum), + BENCHMARK_TYPE(custom_double2, hipcub::Min), #endif CREATE_BENCHMARKS(hipcub::ArgMin), #ifdef HIPCUB_ROCPRIM_API diff --git a/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp b/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp index 11121bfdb218..db69075e7cef 100644 --- a/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -46,8 +46,8 @@ void run_sort_keys_benchmark(benchmark::State& state, size_t&, const key_type*, key_type*, - int64_t, - int64_t, + int, + int, offset_type*, offset_type*, hipStream_t); @@ -181,8 +181,8 @@ void run_sort_pairs_benchmark(benchmark::State& state, key_type*, const value_type*, value_type*, - int64_t, - int64_t, + int, + int, offset_type*, offset_type*, hipStream_t); diff --git a/projects/hipcub/benchmark/benchmark_device_select.cpp b/projects/hipcub/benchmark/benchmark_device_select.cpp index 835789b6ffe0..04097eca9be6 100644 --- a/projects/hipcub/benchmark/benchmark_device_select.cpp +++ b/projects/hipcub/benchmark/benchmark_device_select.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -320,7 +320,7 @@ void run_unique_benchmark(benchmark::State& state, const hipStream_t stream, float discontinuity_probability) { - benchmark_utils::plus op{}; + hipcub::Sum op; std::vector input(size); { @@ -409,7 +409,7 @@ void run_unique_by_key_benchmark(benchmark::State& state, const hipStream_t stream, float discontinuity_probability) { - benchmark_utils::plus op{}; + hipcub::Sum op; std::vector input_keys(size); { diff --git a/projects/hipcub/benchmark/benchmark_device_spmv.cpp b/projects/hipcub/benchmark/benchmark_device_spmv.cpp new file mode 100644 index 000000000000..fcdb1ab9be9c --- /dev/null +++ b/projects/hipcub/benchmark/benchmark_device_spmv.cpp @@ -0,0 +1,269 @@ +// MIT License +// +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "common_benchmark_header.hpp" + +// HIP API +#include + +#ifndef DEFAULT_N +const size_t DEFAULT_N = 1024 * 32; +#endif + +const unsigned int batch_size = 10; +const unsigned int warmup_size = 5; + +template +void run_benchmark(benchmark::State& state, + size_t size, + const hipStream_t stream, + float probability) +{ + const T rand_min = T(1); + const T rand_max = T(10); + + // generate a lexicograhically sorted list of (row, column) index tuples + // number of nonzeroes cannot be guaranteed as duplicates may exist + const int num_nonzeroes_attempt = static_cast( + std::min(static_cast(INT_MAX), + static_cast(probability * static_cast(size * size)))); + std::vector> indices(num_nonzeroes_attempt); + { + std::vector flat_indices + = benchmark_utils::get_random_data(2 * num_nonzeroes_attempt, + 0, + size - 1, + 2 * num_nonzeroes_attempt); + for(int i = 0; i < num_nonzeroes_attempt; i++) + { + indices[i] = std::make_pair(flat_indices[2 * i], flat_indices[2 * i + 1]); + } + std::sort(indices.begin(), indices.end()); + } + + // generate the compressed sparse rows matrix + std::pair prev_cell = std::make_pair(-1, -1); + int num_nonzeroes = 0; + std::vector row_offsets(size + 1); + // this vector might be too large, but doing the allocation now eliminates a + // scan + std::vector column_indices(num_nonzeroes_attempt); + row_offsets[0] = 0; + int last_row_written = 0; + for(int i = 0; i < num_nonzeroes_attempt; i++) + { + if(indices[i] != prev_cell) + { + // update the row offets if we go to the next row (or skip some) + if(indices[i].first != last_row_written) + { + for(int j = last_row_written + 1; j <= indices[i].first; j++) + { + row_offsets[j] = num_nonzeroes; + } + last_row_written = indices[i].first; + } + + column_indices[num_nonzeroes++] = indices[i].second; + + prev_cell = indices[i]; + } + } + // fill in the entries for any missing rows + for(int j = last_row_written + 1; j < static_cast(size) + 1; j++) + { + row_offsets[j] = num_nonzeroes; + } + + // generate the random data once the actual number of nonzeroes are known + std::vector values = benchmark_utils::get_random_data(num_nonzeroes, rand_min, rand_max); + + std::vector vector_x = benchmark_utils::get_random_data(size, rand_min, rand_max); + + T* d_values; + int* d_row_offsets; + int* d_column_indices; + T* d_vector_x; + T* d_vector_y; + HIP_CHECK(hipMalloc(&d_values, values.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_row_offsets, row_offsets.size() * sizeof(int))); + HIP_CHECK(hipMalloc(&d_column_indices, num_nonzeroes * sizeof(int))); + HIP_CHECK(hipMalloc(&d_vector_x, vector_x.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_vector_y, size * sizeof(T))); + HIP_CHECK(hipMemcpy(d_values, values.data(), values.size() * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_row_offsets, + row_offsets.data(), + row_offsets.size() * sizeof(int), + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_column_indices, + column_indices.data(), + num_nonzeroes * sizeof(int), + hipMemcpyHostToDevice)); + HIP_CHECK( + hipMemcpy(d_vector_x, vector_x.data(), vector_x.size() * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + // Allocate temporary storage memory + size_t temp_storage_size_bytes; + + // Get size of d_temp_storage + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + HIP_CHECK(hipcub::DeviceSpmv::CsrMV(nullptr, + temp_storage_size_bytes, + d_values, + d_row_offsets, + d_column_indices, + d_vector_x, + d_vector_y, + size, + size, + num_nonzeroes, + stream)); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + HIP_CHECK(hipDeviceSynchronize()); + + // allocate temporary storage + void* d_temp_storage = nullptr; + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + // Warm-up + for(size_t i = 0; i < warmup_size; i++) + { + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, + temp_storage_size_bytes, + d_values, + d_row_offsets, + d_column_indices, + d_vector_x, + d_vector_y, + size, + size, + num_nonzeroes, + stream)); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + } + HIP_CHECK(hipDeviceSynchronize()); + + for(auto _ : state) + { + auto start = std::chrono::high_resolution_clock::now(); + for(size_t i = 0; i < batch_size; i++) + { + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, + temp_storage_size_bytes, + d_values, + d_row_offsets, + d_column_indices, + d_vector_x, + d_vector_y, + size, + size, + num_nonzeroes, + stream)); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + } + HIP_CHECK(hipDeviceSynchronize()); + + auto end = std::chrono::high_resolution_clock::now(); + auto elapsed_seconds + = std::chrono::duration_cast>(end - start); + state.SetIterationTime(elapsed_seconds.count()); + } + state.SetBytesProcessed(state.iterations() * batch_size * (num_nonzeroes + size) * sizeof(T)); + state.SetItemsProcessed(state.iterations() * batch_size * (num_nonzeroes + size)); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_vector_y)); + HIP_CHECK(hipFree(d_vector_x)); + HIP_CHECK(hipFree(d_column_indices)); + HIP_CHECK(hipFree(d_row_offsets)); + HIP_CHECK(hipFree(d_values)); + HIP_CHECK(hipDeviceSynchronize()); +} + +#define CREATE_BENCHMARK(T, p) \ + benchmark::RegisterBenchmark( \ + std::string("device_spmv_CsrMV.").c_str(), \ + &run_benchmark, \ + size, \ + stream, \ + p) + +#define BENCHMARK_TYPE(type) \ + CREATE_BENCHMARK(type, 1.0e-6f), CREATE_BENCHMARK(type, 1.0e-5f), \ + CREATE_BENCHMARK(type, 1.0e-4f), CREATE_BENCHMARK(type, 1.0e-3f), \ + CREATE_BENCHMARK(type, 1.0e-2f) + +int main(int argc, char* argv[]) +{ + cli::Parser parser(argc, argv); + parser.set_optional("size", "size", DEFAULT_N, "number of values"); + parser.set_optional("trials", "trials", -1, "number of iterations"); + parser.run_and_exit_if_error(); + + // Parse argv + benchmark::Initialize(&argc, argv); + const size_t size = parser.get("size"); + const int trials = parser.get("trials"); + + // HIP + hipStream_t stream = 0; // default + hipDeviceProp_t devProp; + int device_id = 0; + HIP_CHECK(hipGetDevice(&device_id)); + HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); + + std::cout << "benchmark_device_spmv" << std::endl; + std::cout << "[HIP] Device name: " << devProp.name << std::endl; + + // Add benchmarks + std::vector benchmarks = { + BENCHMARK_TYPE(int), + BENCHMARK_TYPE(unsigned int), + BENCHMARK_TYPE(float), + BENCHMARK_TYPE(double), + }; + + // Use manual timing + for(auto& b : benchmarks) + { + b->UseManualTime(); + b->Unit(benchmark::kMillisecond); + } + + // Force number of iterations + if(trials > 0) + { + for(auto& b : benchmarks) + { + b->Iterations(trials); + } + } + + // Run benchmarks + benchmark::RunSpecifiedBenchmarks(); + + return 0; +} diff --git a/projects/hipcub/benchmark/benchmark_utils.hpp b/projects/hipcub/benchmark/benchmark_utils.hpp index da6db74963a0..489cf8de8525 100644 --- a/projects/hipcub/benchmark/benchmark_utils.hpp +++ b/projects/hipcub/benchmark/benchmark_utils.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -36,16 +36,13 @@ #endif #include -#include #ifndef HIPCUB_CUB_API #define HIPCUB_WARP_THREADS_MACRO warpSize #else - #define HIPCUB_WARP_THREADS_MACRO warp_threads + #define HIPCUB_WARP_THREADS_MACRO CUB_PTX_WARP_THREADS #endif -#include _HIPCUB_STD_INCLUDE(limits) - namespace benchmark_utils { const size_t default_max_random_size = 1024 * 1024; @@ -62,11 +59,11 @@ inline auto std::uniform_int_distribution distribution(min, max); std::vector data(size); std::generate(data.begin(), - data.begin() + _HIPCUB_STD::min(size, max_random_size), + data.begin() + std::min(size, max_random_size), [&]() { return distribution(gen); }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(data.begin(), _HIPCUB_STD::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); } return data; } @@ -81,11 +78,11 @@ inline auto std::uniform_real_distribution distribution(min, max); std::vector data(size); std::generate(data.begin(), - data.begin() + _HIPCUB_STD::min(size, max_random_size), + data.begin() + std::min(size, max_random_size), [&]() { return distribution(gen); }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(data.begin(), _HIPCUB_STD::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); } return data; } @@ -99,11 +96,11 @@ inline std::vector std::bernoulli_distribution distribution(p); std::vector data(size); std::generate(data.begin(), - data.begin() + _HIPCUB_STD::min(size, max_random_size), + data.begin() + std::min(size, max_random_size), [&]() { return distribution(gen); }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(data.begin(), _HIPCUB_STD::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); } return data; } @@ -117,12 +114,12 @@ inline T get_random_value(T min, T max) // Can't use std::prefix_sum for inclusive/exclusive scan, because // it does not handle short[] -> int(int a, int b) { a + b; } -> int[] // they way we expect. That's because sum in std::prefix_sum's implementation -// is of type ::hipcub::detail::it_value_t (short) +// is of type typename std::iterator_traits::value_type (short) template OutputIt host_inclusive_scan(InputIt first, InputIt last, OutputIt d_first, BinaryOperation op) { - using input_type = ::hipcub::detail::it_value_t; - using output_type = ::hipcub::detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; + using output_type = typename std::iterator_traits::value_type; using result_type = typename std::conditional::value, input_type, output_type>::type; @@ -144,8 +141,8 @@ template OutputIt host_exclusive_scan( InputIt first, InputIt last, T initial_value, OutputIt d_first, BinaryOperation op) { - using input_type = ::hipcub::detail::it_value_t; - using output_type = ::hipcub::detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; + using output_type = typename std::iterator_traits::value_type; using result_type = typename std::conditional::value, input_type, output_type>::type; @@ -178,8 +175,8 @@ OutputIt host_exclusive_scan_by_key(InputIt first, BinaryOperation op, KeyCompare key_compare_op) { - using input_type = ::hipcub::detail::it_value_t; - using output_type = ::hipcub::detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; + using output_type = typename std::iterator_traits::value_type; using result_type = typename std::conditional::value, input_type, output_type>::type; @@ -306,11 +303,11 @@ struct generate_limits::value>> { static inline T min() { - return _HIPCUB_STD::numeric_limits::min(); + return std::numeric_limits::min(); } static inline T max() { - return _HIPCUB_STD::numeric_limits::max(); + return std::numeric_limits::max(); } }; @@ -360,7 +357,8 @@ inline auto get_random_data(size_t size, T min, T max, size_t max_random_size = template inline auto get_random_data(size_t size, T min, T max, size_t max_random_size = 1024 * 1024) -> - typename std::enable_if::value && !std::is_same_v, + typename std::enable_if::value + && !std::is_same::value, std::vector>::type { @@ -385,15 +383,14 @@ std::vector using key_distribution_type = std::conditional_t::value, std::uniform_int_distribution, std::uniform_real_distribution>; - key_distribution_type key_distribution(_HIPCUB_STD::numeric_limits::max()); + key_distribution_type key_distribution(std::numeric_limits::max()); std::vector keys(size); size_t keys_start_index = 0; while(keys_start_index < size) { const size_t new_segment_length = segment_length_distribution(prng); - const size_t new_segment_end - = _HIPCUB_STD::min(size, keys_start_index + new_segment_length); + const size_t new_segment_end = std::min(size, keys_start_index + new_segment_length); const T key = key_distribution(prng); std::fill(std::next(keys.begin(), keys_start_index), std::next(keys.begin(), new_segment_end), @@ -420,30 +417,26 @@ inline constexpr bool is_power_of_two(const T x) return (x > 0) && ((x & (x - 1)) == 0); } +template +using it_value_t = typename std::iterator_traits::value_type; + using engine_type = std::default_random_engine; // generate_random_data_n() generates only part of sequence and replicates it, // because benchmarks usually do not need "true" random sequence. template -inline auto generate_random_data_n(OutputIter it, - size_t size, - U min, - V max, - Generator& gen, - size_t max_random_size = 1024 * 1024) -> - typename std::enable_if_t>::value, - OutputIter> +inline auto generate_random_data_n( + OutputIter it, size_t size, U min, V max, Generator& gen, size_t max_random_size = 1024 * 1024) + -> typename std::enable_if_t>::value, OutputIter> { - using T = ::hipcub::detail::it_value_t; + using T = it_value_t; using dis_type = typename std::conditional<(sizeof(T) == 1), short, T>::type; std::uniform_int_distribution distribution((T)min, (T)max); - std::generate_n(it, - _HIPCUB_STD::min(size, max_random_size), - [&]() { return distribution(gen); }); + std::generate_n(it, std::min(size, max_random_size), [&]() { return distribution(gen); }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(it, _HIPCUB_STD::min(size - i, max_random_size), it + i); + std::copy_n(it, std::min(size - i, max_random_size), it + i); } return it + size; } @@ -455,23 +448,20 @@ inline auto generate_random_data_n(OutputIterator it, V max, Generator& gen, size_t max_random_size = 1024 * 1024) - -> std::enable_if_t>::value, - OutputIterator> + -> std::enable_if_t>::value, OutputIterator> { - using T = ::hipcub::detail::it_value_t; + using T = typename std::iterator_traits::value_type; std::uniform_real_distribution distribution((T)min, (T)max); - std::generate_n(it, - _HIPCUB_STD::min(size, max_random_size), - [&]() { return distribution(gen); }); + std::generate_n(it, std::min(size, max_random_size), [&]() { return distribution(gen); }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(it, _HIPCUB_STD::min(size - i, max_random_size), it + i); + std::copy_n(it, std::min(size - i, max_random_size), it + i); } return it + size; } -template +template struct alignas(Alignment) custom_aligned_type { unsigned char data[Size]; @@ -485,26 +475,6 @@ inline constexpr auto ceiling_div(const T a, const U b) return a / b + (a % b > 0 ? 1 : 0); } -struct minimum -{ - template - HIPCUB_HOST_DEVICE - auto operator()(const T& a, const U& b) const - { - return a < b ? a : b; - } -}; - -struct plus -{ - template - HIPCUB_HOST_DEVICE - constexpr auto operator()(const A& a, const B& b) const -> decltype(a + b) - { - return a + b; - } -}; - } // namespace benchmark_utils // Need for hipcub::DeviceReduce::Min/Max etc. @@ -518,17 +488,17 @@ class numeric_limits> public: static constexpr inline T min() { - return _HIPCUB_STD::numeric_limits::min(); + return std::numeric_limits::min(); } static constexpr inline T max() { - return _HIPCUB_STD::numeric_limits::max(); + return std::numeric_limits::max(); } static constexpr inline T lowest() { - return _HIPCUB_STD::numeric_limits::lowest(); + return std::numeric_limits::lowest(); } }; @@ -540,17 +510,17 @@ class numeric_limits> public: static constexpr inline T min() { - return _HIPCUB_STD::numeric_limits::min(); + return std::numeric_limits::min(); } static constexpr inline T max() { - return _HIPCUB_STD::numeric_limits::max(); + return std::numeric_limits::max(); } static constexpr inline T lowest() { - return _HIPCUB_STD::numeric_limits::lowest(); + return std::numeric_limits::lowest(); } }; } // namespace std diff --git a/projects/hipcub/benchmark/benchmark_warp_exchange.cpp b/projects/hipcub/benchmark/benchmark_warp_exchange.cpp index d780e1a2c53f..0c41be0588ad 100644 --- a/projects/hipcub/benchmark/benchmark_warp_exchange.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -41,13 +41,17 @@ __device__ auto warp_exchange_benchmark(T* d_output) -> std::enable_if_t> { T thread_data[ItemsPerThread]; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned i = 0; i < ItemsPerThread; ++i) { thread_data[i] = static_cast(i); } - using WarpExchangeT = ::hipcub::WarpExchange; + using WarpExchangeT = ::hipcub::WarpExchange; constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; __shared__ typename WarpExchangeT::TempStorage temp_storage[warps_in_block]; const unsigned warp_id = threadIdx.x / LogicalWarpSize; @@ -55,7 +59,7 @@ __device__ auto warp_exchange_benchmark(T* d_output) WarpExchangeT warp_exchange(temp_storage[warp_id]); Op{}(warp_exchange, thread_data); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned i = 0; i < ItemsPerThread; ++i) { const unsigned global_idx = (BlockSize * blockIdx.x + threadIdx.x) * ItemsPerThread + i; @@ -95,7 +99,7 @@ __device__ auto warp_exchange_scatter_to_striped_benchmark(T* d_output) const unsigned warp_id = threadIdx.x / LogicalWarpSize; T thread_data[ItemsPerThread]; OffsetT thread_ranks[ItemsPerThread]; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned i = 0; i < ItemsPerThread; ++i) { thread_data[i] = static_cast(i); @@ -108,7 +112,7 @@ __device__ auto warp_exchange_scatter_to_striped_benchmark(T* d_output) WarpExchangeT(temp_storage[warp_id]).ScatterToStriped(thread_data, thread_ranks); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned i = 0; i < ItemsPerThread; ++i) { const unsigned striped_global_idx diff --git a/projects/hipcub/benchmark/benchmark_warp_load.cpp b/projects/hipcub/benchmark/benchmark_warp_load.cpp index 95525f335b15..2c74609dfc95 100644 --- a/projects/hipcub/benchmark/benchmark_warp_load.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_load.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -50,7 +50,7 @@ __device__ auto warp_load_benchmark(T* d_input, T* d_output) WarpLoadT(temp_storage[warp_id]).Load(d_input + global_warp_id * tile_size, thread_data); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned i = 0; i < ItemsPerThread; ++i) { const unsigned striped_global_idx diff --git a/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp b/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp index ba35c1c0e439..f6d91fe7f9a2 100644 --- a/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -149,7 +149,7 @@ __global__ template struct max_value { - static constexpr T value = _HIPCUB_STD::numeric_limits::max(); + static constexpr T value = std::numeric_limits::max(); }; template; __shared__ typename wreduce_t::TempStorage storage; - auto reduce_op = benchmark_utils::plus{}; - _CCCL_PRAGMA_NOUNROLL() + auto reduce_op = hipcub::Sum(); +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { value = wreduce_t(storage).Reduce(value, reduce_op); @@ -72,7 +72,7 @@ __device__ auto segmented_warp_reduce_benchmark(const T* d_input, Flag* d_flags, using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { value = wreduce_t(storage).HeadSegmentedSum(value, flag); diff --git a/projects/hipcub/benchmark/benchmark_warp_scan.cpp b/projects/hipcub/benchmark/benchmark_warp_scan.cpp index f9f80baac70d..db3fe941f6cb 100644 --- a/projects/hipcub/benchmark/benchmark_warp_scan.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,7 @@ #include "common_benchmark_header.hpp" -// HIP +// HIP API #include #ifndef DEFAULT_N @@ -56,8 +56,8 @@ struct inclusive_scan using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage; - auto scan_op = benchmark_utils::plus{}; - _CCCL_PRAGMA_NOUNROLL() + auto scan_op = hipcub::Sum(); +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { wscan_t(storage).InclusiveScan(value, value, scan_op); @@ -85,8 +85,8 @@ struct exclusive_scan using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage; - auto scan_op = benchmark_utils::plus{}; - _CCCL_PRAGMA_NOUNROLL() + auto scan_op = hipcub::Sum(); +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { wscan_t(storage).ExclusiveScan(value, value, init, scan_op); @@ -118,7 +118,7 @@ struct broadcast using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage; - _CCCL_PRAGMA_NOUNROLL() +#pragma nounroll for(unsigned int trial = 0; trial < Trials; trial++) { value = wscan_t(storage).Broadcast(value, src_lane); @@ -228,8 +228,9 @@ template auto add_benchmarks(std::vector& benchmarks, const std::string& method_name, hipStream_t stream, - size_t size) -> std::enable_if_t - || std::is_same_v> + size_t size) + -> std::enable_if_t::value + || std::is_same::value> { using custom_double2 = benchmark_utils::custom_type; using custom_int_double = benchmark_utils::custom_type; @@ -248,7 +249,7 @@ template auto add_benchmarks(std::vector& benchmarks, const std::string& method_name, hipStream_t stream, - size_t size) -> std::enable_if_t> + size_t size) -> std::enable_if_t::value> { using custom_double2 = benchmark_utils::custom_type; using custom_int_double = benchmark_utils::custom_type; diff --git a/projects/hipcub/benchmark/benchmark_warp_store.cpp b/projects/hipcub/benchmark/benchmark_warp_store.cpp index 8d846b460093..6632faf178be 100644 --- a/projects/hipcub/benchmark/benchmark_warp_store.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_store.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -40,7 +40,7 @@ __device__ auto warp_store_benchmark(T* d_output) -> std::enable_if_t> { T thread_data[ItemsPerThread]; - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned i = 0; i < ItemsPerThread; ++i) { thread_data[i] = static_cast(i); diff --git a/projects/hipcub/benchmark/common_benchmark_header.hpp b/projects/hipcub/benchmark/common_benchmark_header.hpp index 093a0079ef1b..a632840a815f 100644 --- a/projects/hipcub/benchmark/common_benchmark_header.hpp +++ b/projects/hipcub/benchmark/common_benchmark_header.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -22,9 +22,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -41,11 +43,6 @@ // HIP API #include -#include - -#include _HIPCUB_LIBCXX_INCLUDE(cmath) -#include _HIPCUB_STD_INCLUDE(limits) - // benchmark_utils.hpp should only be included by this header. // The following definition is used as guard in benchmark_utils.hpp // Including benchmark_utils.hpp by itself will cause a compile error. diff --git a/projects/hipcub/cmake/Dependencies.cmake b/projects/hipcub/cmake/Dependencies.cmake index e5d758b30a7b..246a98af0b2e 100644 --- a/projects/hipcub/cmake/Dependencies.cmake +++ b/projects/hipcub/cmake/Dependencies.cmake @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -373,7 +373,7 @@ endif(USER_BUILD_BENCHMARK) # CUB (only for CUDA platform) if(HIP_COMPILER STREQUAL "nvcc") - set(CCCL_MINIMUM_VERSION 3.0.0) + set(CCCL_MINIMUM_VERSION 2.8.2) if(NOT DOWNLOAD_CUB) find_package(CCCL ${CCCL_MINIMUM_VERSION} CONFIG) endif() diff --git a/projects/hipcub/cmake/SetupNVCC.cmake b/projects/hipcub/cmake/SetupNVCC.cmake index 48b59eb2f403..46b88d278332 100644 --- a/projects/hipcub/cmake/SetupNVCC.cmake +++ b/projects/hipcub/cmake/SetupNVCC.cmake @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2018-2026 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -123,6 +123,6 @@ if (NOT _HIPCUB_HIP_NVCC_FLAGS_SET) set(_HIPCUB_HIP_NVCC_FLAGS_SET ON CACHE INTERNAL "") endif() -# Ignore warnings about _CCCL_PRAGMA_UNROLL_FULL() +# Ignore warnings about #pragma unroll # and about deprecated CUDA function(s) used in hip/nvcc_detail/hip_runtime_api.h # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${HIP_CPP_CONFIG_FLAGS_STRIP} -Wno-unknown-pragmas -Wno-deprecated-declarations" CACHE STRING "compile flags" FORCE) diff --git a/projects/hipcub/examples/device/example_device_partition_flagged.cpp b/projects/hipcub/examples/device/example_device_partition_flagged.cpp index 35e4f529c866..f56cae1d6e8d 100644 --- a/projects/hipcub/examples/device/example_device_partition_flagged.cpp +++ b/projects/hipcub/examples/device/example_device_partition_flagged.cpp @@ -82,10 +82,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = _HIPCUB_STD::max(1, repeat); + repeat = std::max(1, repeat); int j = i; - while(j < _HIPCUB_STD::min(i + repeat, num_items)) + while (j < std::min(i + repeat, num_items)) { h_flags[j] = 0; h_in[j] = key; diff --git a/projects/hipcub/examples/device/example_device_partition_if.cpp b/projects/hipcub/examples/device/example_device_partition_if.cpp index f12c8188f373..7516f9a40d77 100644 --- a/projects/hipcub/examples/device/example_device_partition_if.cpp +++ b/projects/hipcub/examples/device/example_device_partition_if.cpp @@ -93,10 +93,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = _HIPCUB_STD::max(1, repeat); + repeat = std::max(1, repeat); int j = i; - while(j < _HIPCUB_STD::min(i + repeat, num_items)) + while (j < std::min(i + repeat, num_items)) { h_in[j] = key; j++; diff --git a/projects/hipcub/examples/device/example_device_select_flagged.cpp b/projects/hipcub/examples/device/example_device_select_flagged.cpp index ba52705f01b0..75c39789deeb 100644 --- a/projects/hipcub/examples/device/example_device_select_flagged.cpp +++ b/projects/hipcub/examples/device/example_device_select_flagged.cpp @@ -83,10 +83,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = _HIPCUB_STD::max(1, repeat); + repeat = std::max(1, repeat); int j = i; - while(j < _HIPCUB_STD::min(i + repeat, num_items)) + while (j < std::min(i + repeat, num_items)) { h_flags[j] = 0; h_in[j] = key; diff --git a/projects/hipcub/examples/device/example_device_select_if.cpp b/projects/hipcub/examples/device/example_device_select_if.cpp index 0493d3c0c476..40a81bc15bc7 100644 --- a/projects/hipcub/examples/device/example_device_select_if.cpp +++ b/projects/hipcub/examples/device/example_device_select_if.cpp @@ -93,10 +93,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = _HIPCUB_STD::max(1, repeat); + repeat = std::max(1, repeat); int j = i; - while(j < _HIPCUB_STD::max(i + repeat, num_items)) + while (j < std::max(i + repeat, num_items)) { h_in[j] = key; j++; diff --git a/projects/hipcub/examples/device/example_device_select_unique.cpp b/projects/hipcub/examples/device/example_device_select_unique.cpp index 196850ed2f9d..d923b29cdb4d 100644 --- a/projects/hipcub/examples/device/example_device_select_unique.cpp +++ b/projects/hipcub/examples/device/example_device_select_unique.cpp @@ -80,10 +80,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = _HIPCUB_STD::max(1, repeat); + repeat = std::max(1, repeat); int j = i; - while(j < _HIPCUB_STD::min(i + repeat, num_items)) + while (j < std::min(i + repeat, num_items)) { h_in[j] = key; j++; diff --git a/projects/hipcub/examples/example_utils.hpp b/projects/hipcub/examples/example_utils.hpp index ef6ea06bc3b6..deedbc937add 100644 --- a/projects/hipcub/examples/example_utils.hpp +++ b/projects/hipcub/examples/example_utils.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,10 +34,9 @@ #include #include -#include #include - -#include _HIPCUB_STD_INCLUDE(functional) +#include +#include #define AssertEquals(a, b) if ((a) != (b)) { std::cerr << "\n(" << __FILE__ << ": " << __LINE__ << ")\n"; exit(1);} @@ -76,8 +75,8 @@ struct CommandLineArgs std::vector args; hipDeviceProp_t deviceProp; float device_giga_bandwidth; - size_t device_free_physmem; - size_t device_total_physmem; + std::size_t device_free_physmem; + std::size_t device_total_physmem; /** * Constructor @@ -125,7 +124,7 @@ struct CommandLineArgs { using namespace std; - for(size_t i = 0; i < keys.size(); ++i) + for (std::size_t i = 0; i < keys.size(); ++i) { if (keys[i] == string(arg_name)) return true; @@ -147,8 +146,8 @@ struct CommandLineArgs /** * Returns the commandline parameter for a given index (not including flags) */ - template - void GetCmdLineArgument(size_t index, T& val) + template + void GetCmdLineArgument(std::size_t index, T &val) { using namespace std; if (index < args.size()) { @@ -165,7 +164,7 @@ struct CommandLineArgs { using namespace std; - for(size_t i = 0; i < keys.size(); ++i) + for (std::size_t i = 0; i < keys.size(); ++i) { if (keys[i] == string(arg_name)) { @@ -190,7 +189,7 @@ struct CommandLineArgs vals.clear(); // Recover from multi-value string - for(size_t i = 0; i < keys.size(); ++i) + for (std::size_t i = 0; i < keys.size(); ++i) { if (keys[i] == string(arg_name)) { @@ -385,6 +384,7 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos return 0; } + // /** // * Verify the contents of a device array match those // * of a host array @@ -392,7 +392,7 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos // int CompareDeviceResults( // hipcub::NullType */* h_reference */, // hipcub::NullType */* d_data */, -// size_t /* num_items */, +// std::size_t /* num_items */, // bool /* verbose */ = true, // bool /* display_data */ = false) // { @@ -407,7 +407,7 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos // int CompareDeviceResults( // S *h_reference, // rocprim::discard_iterator d_data, -// size_t num_items, +// std::size_t num_items, // bool verbose = true, // bool display_data = false) // { @@ -418,9 +418,13 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos * Verify the contents of a device array match those * of a host array */ -template +template int CompareDeviceResults( - S* h_reference, T* d_data, size_t num_items, bool verbose = true, bool display_data = false) + S *h_reference, + T *d_data, + std::size_t num_items, + bool verbose = true, + bool display_data = false) { // Allocate array on host T *h_data = (T*) malloc(num_items * sizeof(T)); @@ -432,12 +436,12 @@ int CompareDeviceResults( if (display_data) { printf("Reference:\n"); - for(size_t i = 0; i < num_items; i++) + for (std::size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_reference[i]) << ", "; } printf("\n\nComputed:\n"); - for(size_t i = 0; i < num_items; i++) + for (std::size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_data[i]) << ", "; } @@ -458,9 +462,13 @@ int CompareDeviceResults( * Verify the contents of a device array match those * of a device array */ -template +template int CompareDeviceDeviceResults( - T* d_reference, T* d_data, size_t num_items, bool verbose = true, bool display_data = false) + T *d_reference, + T *d_data, + std::size_t num_items, + bool verbose = true, + bool display_data = false) { // Allocate array on host T *h_reference = (T*) malloc(num_items * sizeof(T)); @@ -473,12 +481,12 @@ int CompareDeviceDeviceResults( // Display data if (display_data) { printf("Reference:\n"); - for(size_t i = 0; i < num_items; i++) + for (std::size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_reference[i]) << ", "; } printf("\n\nComputed:\n"); - for(size_t i = 0; i < num_items; i++) + for (std::size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_data[i]) << ", "; } @@ -498,11 +506,13 @@ int CompareDeviceDeviceResults( /** * Print the contents of a host array */ -template -void DisplayResults(InputIteratorT h_data, size_t num_items) +template +void DisplayResults( + InputIteratorT h_data, + std::size_t num_items) { // Display data - for(size_t i = 0; i < num_items; i++) + for (std::size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_data[i]) << ", "; } @@ -563,8 +573,8 @@ void RandomBits( int current_bit = j * WORD_BYTES * 8; unsigned int word = 0xffffffff; - word &= 0xffffffff << _HIPCUB_STD::max(0, begin_bit - current_bit); - word &= 0xffffffff >> _HIPCUB_STD::max(0, (current_bit + (WORD_BYTES * 8)) - end_bit); + word &= 0xffffffff << std::max(0, begin_bit - current_bit); + word &= 0xffffffff >> std::max(0, (current_bit + (WORD_BYTES * 8)) - end_bit); for (int i = 0; i <= entropy_reduction; i++) { @@ -579,7 +589,7 @@ void RandomBits( memcpy(&key, word_buff, sizeof(K)); K copy = key; - if constexpr(std::is_floating_point::value) + if HIPCUB_IF_CONSTEXPR(std::is_floating_point::value) #ifndef _WIN32 if(!std::isnan(copy)) #else diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp index f05546715605..8e223cbbf475 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -46,9 +46,10 @@ using BlockScanRunningPrefixOp = cub::BlockScanRunningPrefixOp; template> using TilePrefixCallbackOp - = cub::TilePrefixCallbackOp; + = cub::TilePrefixCallbackOp; template // IWYU pragma: export -#include - BEGIN_HIPCUB_NAMESPACE struct DeviceAdjacentDifference { template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractLeftCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeftCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractLeftCopy(d_temp_storage, @@ -63,16 +60,39 @@ struct DeviceAdjacentDifference stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractLeftCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SubtractLeftCopy(d_temp_storage, + temp_storage_bytes, + d_input, + d_output, + num_items, + difference_op, + stream); + } + template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractLeft(void* d_temp_storage, - size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeft(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractLeft(d_temp_storage, @@ -83,18 +103,38 @@ struct DeviceAdjacentDifference stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractLeft(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SubtractLeft(d_temp_storage, + temp_storage_bytes, + d_input, + num_items, + difference_op, + stream); + } + template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractRightCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRightCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractRightCopy(d_temp_storage, @@ -106,16 +146,39 @@ struct DeviceAdjacentDifference stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractRightCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SubtractRightCopy(d_temp_storage, + temp_storage_bytes, + d_input, + d_output, + num_items, + difference_op, + stream); + } + template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractRight(void* d_temp_storage, - size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRight(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractRight(d_temp_storage, @@ -125,6 +188,27 @@ struct DeviceAdjacentDifference difference_op, stream)); } + + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractRight(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SubtractRight(d_temp_storage, + temp_storage_bytes, + d_input, + num_items, + difference_op, + stream); + } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp index 64f22a982d59..a6315c517241 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,13 +40,13 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceCopy { template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - _HIPCUB_STD::int64_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + uint32_t num_buffers, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceCopy::Batched(d_temp_storage, temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp index 7bc48bafdde2..0f22c405179d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,11 +33,13 @@ #include // IWYU pragma: export -#include +#if __cccl_lib_mdspan + #include BEGIN_HIPCUB_NAMESPACE template using extents = ::cuda::std::extents; END_HIPCUB_NAMESPACE +#endif // __cccl_lib_mdspan BEGIN_HIPCUB_NAMESPACE @@ -162,7 +164,8 @@ HIPCUB_RUNTIME_FUNCTION cub::DeviceFor::Bulk(d_temp_storage, temp_storage_bytes, shape, op, stream)); } - // ForEachInExtents only enables when the cccl mdspan extension is enabled +// ForEachInExtents only enables when the cccl mdspan extension is enabled +#ifdef __cccl_lib_mdspan template HIPCUB_RUNTIME_FUNCTION static hipError_t ForEachInExtents(void* d_temp_storage, @@ -186,6 +189,7 @@ HIPCUB_RUNTIME_FUNCTION { return hipCUDAErrorTohipError(cub::DeviceFor::ForEachInExtents(extents, op, stream)); } +#endif // __cccl_lib_mdspan }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp index f184c4d2ebd7..60018ad68549 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -61,6 +61,31 @@ struct DeviceHistogram stream)); } + template + HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t + HistogramEven(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram, + int num_levels, + LevelT lower_level, + LevelT upper_level, + OffsetT num_samples, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return HistogramEven(d_temp_storage, + temp_storage_bytes, + d_samples, + d_histogram, + num_levels, + lower_level, + upper_level, + num_samples, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t HistogramEven(void* d_temp_storage, size_t& temp_storage_bytes, @@ -87,6 +112,35 @@ struct DeviceHistogram stream)); } + template + HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t + HistogramEven(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram, + int num_levels, + LevelT lower_level, + LevelT upper_level, + OffsetT num_row_samples, + OffsetT num_rows, + size_t row_stride_bytes, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return HistogramEven(d_temp_storage, + temp_storage_bytes, + d_samples, + d_histogram, + num_levels, + lower_level, + upper_level, + num_row_samples, + num_rows, + row_stride_bytes, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t MultiHistogramEven(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram[NUM_ACTIVE_CHANNELS], - int num_levels[NUM_ACTIVE_CHANNELS], - LevelT lower_level[NUM_ACTIVE_CHANNELS], - LevelT upper_level[NUM_ACTIVE_CHANNELS], - OffsetT num_row_pixels, - OffsetT num_rows, - size_t row_stride_bytes, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t + MultiHistogramEven(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram[NUM_ACTIVE_CHANNELS], + int num_levels[NUM_ACTIVE_CHANNELS], + LevelT lower_level[NUM_ACTIVE_CHANNELS], + LevelT upper_level[NUM_ACTIVE_CHANNELS], + OffsetT num_row_pixels, + OffsetT num_rows, + size_t row_stride_bytes, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceHistogram::MultiHistogramEven( @@ -178,7 +232,7 @@ struct DeviceHistogram num_row_pixels, num_rows, row_stride_bytes, - reinterpret_cast(stream))); + stream)); } template -HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS -HIPCUB_RUNTIME_FUNCTION - static hipError_t MultiHistogramEven(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram[NUM_ACTIVE_CHANNELS], - int num_levels[NUM_ACTIVE_CHANNELS], - LevelT lower_level[NUM_ACTIVE_CHANNELS], - LevelT upper_level[NUM_ACTIVE_CHANNELS], - OffsetT num_row_pixels, - OffsetT num_rows, - size_t row_stride_bytes, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t + MultiHistogramEven(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram[NUM_ACTIVE_CHANNELS], + int num_levels[NUM_ACTIVE_CHANNELS], + LevelT lower_level[NUM_ACTIVE_CHANNELS], + LevelT upper_level[NUM_ACTIVE_CHANNELS], + OffsetT num_row_pixels, + OffsetT num_rows, + size_t row_stride_bytes, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - - return MultiHistogramEven(d_temp_storage, - temp_storage_bytes, - d_samples, - d_histogram, - num_levels, - lower_level, - upper_level, - num_row_pixels, - num_rows, - row_stride_bytes, - stream); + return MultiHistogramEven(d_temp_storage, + temp_storage_bytes, + d_samples, + d_histogram, + num_levels, + lower_level, + upper_level, + num_row_pixels, + num_rows, + row_stride_bytes, + stream); } template @@ -242,6 +289,29 @@ HIPCUB_RUNTIME_FUNCTION stream)); } + template + HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t + HistogramRange(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram, + int num_levels, + LevelT* d_levels, + OffsetT num_samples, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return HistogramRange(d_temp_storage, + temp_storage_bytes, + d_samples, + d_histogram, + num_levels, + d_levels, + num_samples, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t HistogramRange(void* d_temp_storage, size_t& temp_storage_bytes, @@ -266,6 +336,33 @@ HIPCUB_RUNTIME_FUNCTION stream)); } + template + HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t + HistogramRange(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram, + int num_levels, + LevelT* d_levels, + OffsetT num_row_samples, + OffsetT num_rows, + size_t row_stride_bytes, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return HistogramRange(d_temp_storage, + temp_storage_bytes, + d_samples, + d_histogram, + num_levels, + d_levels, + num_row_samples, + num_rows, + row_stride_bytes, + stream); + } + template + HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t + MultiHistogramRange(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram[NUM_ACTIVE_CHANNELS], + int num_levels[NUM_ACTIVE_CHANNELS], + LevelT* d_levels[NUM_ACTIVE_CHANNELS], + OffsetT num_pixels, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return MultiHistogramRange(d_temp_storage, + temp_storage_bytes, + d_samples, + d_histogram, + num_levels, + d_levels, + num_pixels, + stream); + } + template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - _HIPCUB_STD::int64_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + uint32_t num_buffers, + hipStream_t stream = 0) { - if(num_buffers == 0) - { - temp_storage_bytes = 0; - return hipSuccess; - } - return hipCUDAErrorTohipError(::cub::DeviceMemcpy::Batched(d_temp_storage, temp_storage_bytes, input_buffer_it, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp index 61c3b2dde593..f314f5a128ef 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp @@ -1,6 +1,6 @@ /****************************************************************************** - * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2025-2026, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,9 +33,6 @@ #include // IWYU pragma: export -#include // IWYU pragma: export -using ::cuda::std::int64_t; - BEGIN_HIPCUB_NAMESPACE struct DeviceMerge @@ -47,11 +44,11 @@ struct DeviceMerge typename CompareOp = ::cuda::std::less<>> HIPCUB_RUNTIME_FUNCTION static hipError_t MergeKeys(void* d_temp_storage, - size_t& temp_storage_bytes, + std::size_t& temp_storage_bytes, KeyIteratorIn1 keys_in1, - int64_t num_keys1, + int num_keys1, KeyIteratorIn2 keys_in2, - int64_t num_keys2, + int num_keys2, KeyIteratorOut keys_out, CompareOp compare_op = {}, hipStream_t stream = 0) @@ -77,13 +74,13 @@ struct DeviceMerge typename CompareOp = ::cuda::std::less<>> HIPCUB_RUNTIME_FUNCTION static hipError_t MergePairs(void* d_temp_storage, - size_t& temp_storage_bytes, + std::size_t& temp_storage_bytes, KeyIteratorIn1 keys_in1, ValueIteratorIn1 values_in1, - int64_t num_keys1, + int num_keys1, KeyIteratorIn2 keys_in2, ValueIteratorIn2 values_in2, - int64_t num_keys2, + int num_keys2, KeyIteratorOut keys_out, ValueIteratorOut values_out, CompareOp compare_op = {}, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp index c047b61beae3..be8792534de1 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,14 +40,13 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceMergeSort { template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortPairs(d_temp_storage, temp_storage_bytes, @@ -58,22 +57,42 @@ struct DeviceMergeSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairs(d_temp_storage, + temp_storage_bytes, + d_keys, + d_items, + num_items, + compare_op, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - ValueInputIteratorT d_input_items, - KeyIteratorT d_output_keys, - ValueIteratorT d_output_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + ValueInputIteratorT d_input_items, + KeyIteratorT d_output_keys, + ValueIteratorT d_output_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortPairsCopy(d_temp_storage, temp_storage_bytes, @@ -86,14 +105,43 @@ struct DeviceMergeSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + ValueInputIteratorT d_input_items, + KeyIteratorT d_output_keys, + ValueIteratorT d_output_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairsCopy(d_temp_storage, + temp_storage_bytes, + d_input_keys, + d_input_items, + d_output_keys, + d_output_items, + num_items, + compare_op, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -103,18 +151,31 @@ struct DeviceMergeSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items, compare_op, stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortKeysCopy(d_temp_storage, @@ -126,15 +187,39 @@ struct DeviceMergeSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) + + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeysCopy(d_temp_storage, + temp_storage_bytes, + d_input_keys, + d_output_keys, + num_items, + compare_op, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::StableSortPairs(d_temp_storage, temp_storage_bytes, @@ -145,14 +230,34 @@ struct DeviceMergeSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortPairs(d_temp_storage, + temp_storage_bytes, + d_keys, + d_items, + num_items, + compare_op, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::StableSortKeys(d_temp_storage, temp_storage_bytes, @@ -162,18 +267,36 @@ struct DeviceMergeSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortKeys(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + compare_op, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::StableSortKeysCopy(d_temp_storage, temp_storage_bytes, @@ -188,15 +311,15 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysCopy(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp index b7a6f12f99a3..767fd257268c 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -64,6 +64,33 @@ struct DevicePartition stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t Flagged(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + FlagIterator d_flags, + OutputIteratorT d_out, + NumSelectedIteratorT d_num_selected_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Flagged(d_temp_storage, + temp_storage_bytes, + d_in, + d_flags, + d_out, + d_num_selected_out, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t If(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumSelectedIteratorT d_num_selected_out, + NumItemsT num_items, + SelectOp select_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return If(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + d_num_selected_out, + num_items, + select_op, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, - size_t& temp_storage_bytes, + std::size_t& temp_storage_bytes, InputIteratorT d_in, FirstOutputIteratorT d_first_part_out, SecondOutputIteratorT d_second_part_out, @@ -122,6 +176,42 @@ struct DevicePartition select_second_part_op, stream)); } + + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t If(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_in, + FirstOutputIteratorT d_first_part_out, + SecondOutputIteratorT d_second_part_out, + UnselectedOutputIteratorT d_unselected_out, + NumSelectedIteratorT d_num_selected_out, + NumItemsT num_items, + SelectFirstPartOp select_first_part_op, + SelectSecondPartOp select_second_part_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return If(d_temp_storage, + temp_storage_bytes, + d_in, + d_first_part_out, + d_second_part_out, + d_unselected_out, + d_num_selected_out, + num_items, + select_first_part_op, + select_second_part_op, + stream); + } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp index 30dd10b31800..f2b396903480 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -65,6 +65,33 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairs(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortPairs(void* d_temp_storage, size_t& temp_storage_bytes, @@ -135,6 +162,29 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairs(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortPairs(void* d_temp_storage, size_t& temp_storage_bytes, @@ -202,6 +252,33 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -275,6 +352,29 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -320,15 +420,14 @@ struct DeviceRadixSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - NumItemsT num_items, - int begin_bit = 0, - int end_bit = sizeof(KeyT) * 8, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + NumItemsT num_items, + int begin_bit = 0, + int end_bit = sizeof(KeyT) * 8, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -340,6 +439,29 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeys(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -400,6 +522,27 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeys(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -458,6 +601,29 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -518,6 +684,27 @@ struct DeviceRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + NumItemsT num_items, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static auto SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp index b8a5edf87bb2..8d81dce0e1ed 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,14 +32,9 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" -#include "../../../util_type.hpp" #include // IWYU pragma: export -#include - -#include - BEGIN_HIPCUB_NAMESPACE class DeviceReduce @@ -69,6 +64,33 @@ class DeviceReduce stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + ReduceOpT reduction_op, + T init, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Reduce(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_items, + reduction_op, + init, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t Sum(void* d_temp_storage, size_t& temp_storage_bytes, @@ -85,6 +107,20 @@ class DeviceReduce stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t Min(void* d_temp_storage, size_t& temp_storage_bytes, @@ -101,50 +137,30 @@ class DeviceReduce stream)); } - template - HIPCUB_RUNTIME_FUNCTION + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Min(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMin(void* d_temp_storage, size_t& temp_storage_bytes, InputIteratorT d_in, ExtremumOutIteratorT d_min_out, IndexOutIteratorT d_index_out, - NumItemsT num_items, + ::std::int64_t num_items, hipStream_t stream = 0) { - using value_type = ::hipcub::detail::it_value_t; - using index_type = ::hipcub::detail::it_value_t; - - // CUB handles zero-length inputs in its internal dispatch layer. - // That behavior must be reproduced manually, so this case is handled here. - if(num_items == 0) - { - if(d_temp_storage == nullptr) - { - temp_storage_bytes = sizeof(int); - return hipSuccess; - } - - value_type init_value = ::cuda::std::numeric_limits::max(); - index_type init_index = 1; - - hipError_t e1 = hipMemcpyAsync(d_min_out, - &init_value, - sizeof(value_type), - hipMemcpyHostToDevice, - stream); - - hipError_t e2 = hipMemcpyAsync(d_index_out, - &init_index, - sizeof(index_type), - hipMemcpyHostToDevice, - stream); - - return (e1 != hipSuccess ? e1 : e2); - } - return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, @@ -169,40 +185,28 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using pair_type = ::hipcub::detail::it_value_t; - using value_type = decltype(pair_type::value); - using index_type = decltype(pair_type::key); - - if(num_items == 0) - { - if(d_temp_storage == nullptr) - { - temp_storage_bytes = sizeof(int); - return hipSuccess; - } - - pair_type init; - init.key = static_cast(1); - init.value = ::cuda::std::numeric_limits::max(); - - return hipMemcpyAsync(d_out, &init, sizeof(pair_type), hipMemcpyHostToDevice, stream); - } - - pair_type* out_pair = reinterpret_cast(d_out); - - value_type* d_min_out = &(out_pair->value); - index_type* d_index_out = &(out_pair->key); - _CCCL_SUPPRESS_DEPRECATED_PUSH - auto status = ArgMin(d_temp_storage, - temp_storage_bytes, - d_in, - d_min_out, - d_index_out, - static_cast(num_items), - stream); + return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMin(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_items, + stream)); _CCCL_SUPPRESS_DEPRECATED_POP - return status; + } + + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); } template @@ -221,52 +225,30 @@ class DeviceReduce stream)); } - template + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Max(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); + } + + template HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMax(void* d_temp_storage, size_t& temp_storage_bytes, InputIteratorT d_in, ExtremumOutIteratorT d_max_out, IndexOutIteratorT d_index_out, - NumItemsT num_items, - hipStream_t stream = 0) + ::std::int64_t num_items, + hipError_t stream = 0) { - using value_type = ::hipcub::detail::it_value_t; - using index_type = ::hipcub::detail::it_value_t; - - // CUB documentation claims zero-length inputs initialize with numeric_limits::max(), - // but the actual CUB implementation uses numeric_limits::lowest(). - // hipCUB matches the implementation. - - if(num_items == 0) - { - if(d_temp_storage == nullptr) - { - temp_storage_bytes = sizeof(int); - return hipSuccess; - } - - value_type init_value = ::cuda::std::numeric_limits::lowest(); - index_type init_index = 1; // hipCUB 1-based index - - hipError_t e1 = hipMemcpyAsync(d_max_out, - &init_value, - sizeof(value_type), - hipMemcpyHostToDevice, - stream); - - hipError_t e2 = hipMemcpyAsync(d_index_out, - &init_index, - sizeof(index_type), - hipMemcpyHostToDevice, - stream); - - return (e1 != hipSuccess ? e1 : e2); - } - return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, @@ -277,11 +259,13 @@ class DeviceReduce } template - HIPCUB_DEPRECATED_BECAUSE("CUB has superseded this interface in favor of the ArgMax interface " - "that takes two separate iterators: one iterator to which the " - "extremum is written and another " - "iterator to which the index of the found extremum is written. ") -HIPCUB_RUNTIME_FUNCTION + HIPCUB_DEPRECATED_BECAUSE( + "CUB has superseded this interface in favor of the ArgMax interface " + "that takes two separate " + "iterators: one iterator to which the extremum is written and another " + "iterator to which the " + "index of the found extremum is written. ") + HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMax(void* d_temp_storage, size_t& temp_storage_bytes, InputIteratorT d_in, @@ -289,40 +273,28 @@ HIPCUB_RUNTIME_FUNCTION NumItemsT num_items, hipStream_t stream = 0) { - using pair_type = ::hipcub::detail::it_value_t; - using value_type = decltype(pair_type::value); - using index_type = decltype(pair_type::key); - - if(num_items == 0) - { - if(d_temp_storage == nullptr) - { - temp_storage_bytes = sizeof(int); - return hipSuccess; - } - - pair_type init; - init.key = static_cast(1); - init.value = ::cuda::std::numeric_limits::lowest(); - - return hipMemcpyAsync(d_out, &init, sizeof(pair_type), hipMemcpyHostToDevice, stream); - } - - pair_type* out_pair = reinterpret_cast(d_out); - - value_type* d_max_out = &(out_pair->value); - index_type* d_index_out = &(out_pair->key); - _CCCL_SUPPRESS_DEPRECATED_PUSH - auto status = ArgMax(d_temp_storage, - temp_storage_bytes, - d_in, - d_max_out, - d_index_out, - static_cast(num_items), - stream); + return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMax(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_items, + stream)); _CCCL_SUPPRESS_DEPRECATED_POP - return status; + } + + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); } template - struct value_only_pair_output_iterator + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + ReduceByKey(void* d_temp_storage, + size_t& temp_storage_bytes, + KeysInputIteratorT d_keys_in, + UniqueOutputIteratorT d_unique_out, + ValuesInputIteratorT d_values_in, + AggregatesOutputIteratorT d_aggregates_out, + NumRunsOutputIteratorT d_num_runs_out, + ReductionOpT reduction_op, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) { - ScalarOutputIt out; - using value_type = ::cub::KeyValuePair; - HIPCUB_HOST_DEVICE - value_only_pair_output_iterator(ScalarOutputIt o) - : out(o) - {} - HIPCUB_HOST_DEVICE - value_only_pair_output_iterator& operator*() - { - return *this; - } - HIPCUB_HOST_DEVICE - value_only_pair_output_iterator& operator=(value_type const& p) - { - *out = p.value; - return *this; - } - HIPCUB_HOST_DEVICE - value_only_pair_output_iterator& operator++() - { - ++out; - return *this; - } - HIPCUB_HOST_DEVICE - value_only_pair_output_iterator operator++(int) - { - value_only_pair_output_iterator tmp = *this; - ++out; - return tmp; - } - }; + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ReduceByKey(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_unique_out, + d_values_in, + d_aggregates_out, + d_num_runs_out, + reduction_op, + num_items, + stream); + } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp index dbfae1b63b4a..824ec2228745 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -63,6 +63,32 @@ class DeviceRunLengthEncode stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Encode(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + UniqueOutputIteratorT d_unique_out, + LengthsOutputIteratorT d_counts_out, + NumRunsOutputIteratorT d_num_runs_out, + int num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Encode(d_temp_storage, + temp_storage_bytes, + d_in, + d_unique_out, + d_counts_out, + d_num_runs_out, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + NonTrivialRuns(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OffsetsOutputIteratorT d_offsets_out, + LengthsOutputIteratorT d_lengths_out, + NumRunsOutputIteratorT d_num_runs_out, + int num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return NonTrivialRuns(d_temp_storage, + temp_storage_bytes, + d_in, + d_offsets_out, + d_lengths_out, + d_num_runs_out, + num_items, + stream); + } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp index 660a6942d4f3..72ad11f7bc8b 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,12 +32,9 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" -#include "../thread/thread_operators.hpp" #include // IWYU pragma: export -#include - BEGIN_HIPCUB_NAMESPACE class DeviceScan @@ -60,6 +57,20 @@ class DeviceScan stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t InclusiveSum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return InclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSum(void* d_temp_storage, @@ -71,6 +82,23 @@ class DeviceScan return InclusiveSum(d_temp_storage, temp_storage_bytes, d_data, d_data, num_items, stream); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t InclusiveSum(void* d_temp_storage, + size_t& temp_storage_bytes, + IteratorT d_data, + int num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return InclusiveSum(d_temp_storage, + temp_storage_bytes, + d_data, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t InclusiveScan(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + ScanOpT scan_op, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return InclusiveScan(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + scan_op, + num_items, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScan(void* d_temp_storage, @@ -111,6 +163,25 @@ class DeviceScan stream); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t InclusiveScan(void* d_temp_storage, + size_t& temp_storage_bytes, + IteratorT d_data, + ScanOpT scan_op, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return InclusiveScan(d_temp_storage, + temp_storage_bytes, + d_data, + scan_op, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveSum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSum(void* d_temp_storage, @@ -164,6 +249,23 @@ class DeviceScan return ExclusiveSum(d_temp_storage, temp_storage_bytes, d_data, d_data, num_items, stream); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveSum(void* d_temp_storage, + size_t& temp_storage_bytes, + IteratorT d_data, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveSum(d_temp_storage, + temp_storage_bytes, + d_data, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveScan(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + ScanOpT scan_op, + InitValueT init_value, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveScan(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + scan_op, + init_value, + num_items, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScan(void* d_temp_storage, @@ -209,6 +338,27 @@ class DeviceScan stream); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveScan(void* d_temp_storage, + size_t& temp_storage_bytes, + IteratorT d_data, + ScanOpT scan_op, + InitValueT init_value, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveScan(d_temp_storage, + temp_storage_bytes, + d_data, + scan_op, + init_value, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveScan(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + ScanOpT scan_op, + FutureValue init_value, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveScan(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + scan_op, + init_value, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveScan(void* d_temp_storage, + size_t& temp_storage_bytes, + IteratorT d_data, + ScanOpT scan_op, + FutureValue init_value, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveScan(d_temp_storage, + temp_storage_bytes, + d_data, + scan_op, + init_value, + num_items, + stream); + } + template, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = ::cuda::std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -284,13 +487,40 @@ class DeviceScan stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveSumByKey(void* d_temp_storage, + size_t& temp_storage_bytes, + KeysInputIteratorT d_keys_in, + ValuesInputIteratorT d_values_in, + ValuesOutputIteratorT d_values_out, + NumItemsT num_items, + EqualityOpT equality_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveSumByKey(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_values_in, + d_values_out, + num_items, + equality_op, + stream); + } + template, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = ::cuda::std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -318,8 +548,41 @@ class DeviceScan template, - typename NumItemsT = std::uint32_t> + typename ScanOpT, + typename InitValueT, + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = ::cuda::std::uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t ExclusiveScanByKey(void* d_temp_storage, + size_t& temp_storage_bytes, + KeysInputIteratorT d_keys_in, + ValuesInputIteratorT d_values_in, + ValuesOutputIteratorT d_values_out, + ScanOpT scan_op, + InitValueT init_value, + NumItemsT num_items, + EqualityOpT equality_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ExclusiveScanByKey(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_values_in, + d_values_out, + scan_op, + init_value, + num_items, + equality_op, + stream); + } + + template HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -340,12 +603,39 @@ class DeviceScan stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t InclusiveSumByKey(void* d_temp_storage, + size_t& temp_storage_bytes, + KeysInputIteratorT d_keys_in, + ValuesInputIteratorT d_values_in, + ValuesOutputIteratorT d_values_out, + NumItemsT num_items, + EqualityOpT equality_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return InclusiveSumByKey(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_values_in, + d_values_out, + num_items, + equality_op, + stream); + } + template, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = ::cuda::std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -372,8 +662,8 @@ class DeviceScan typename ValuesInputIteratorT, typename ValuesOutputIteratorT, typename ScanOpT, - typename EqualityOpT = ::cuda::std::equal_to<>, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = ::cuda::std::uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp index 8d4807eb72d1..a69e17fee04c 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -69,6 +69,39 @@ struct DeviceSegmentedRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairs(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, size_t& temp_storage_bytes, @@ -95,6 +128,35 @@ struct DeviceSegmentedRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairs(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -126,6 +188,39 @@ struct DeviceSegmentedRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -153,6 +248,35 @@ struct DeviceSegmentedRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -179,6 +303,35 @@ struct DeviceSegmentedRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeys(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -203,6 +356,33 @@ struct DeviceSegmentedRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeys(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -230,6 +410,35 @@ struct DeviceSegmentedRadixSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -254,6 +463,33 @@ struct DeviceSegmentedRadixSort end_bit, stream)); } + + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + int begin_bit, + int end_bit, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + begin_bit, + end_bit, + stream); + } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp index 942643b3bfd6..4f21fbfac373 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -44,17 +44,16 @@ struct DeviceSegmentedReduce typename OffsetIteratorT, typename ReductionOp, typename T> - HIPCUB_RUNTIME_FUNCTION - static hipError_t Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - ReductionOp reduction_op, - T initial_value, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + ReductionOp reduction_op, + T initial_value, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Reduce(d_temp_storage, temp_storage_bytes, @@ -68,16 +67,46 @@ struct DeviceSegmentedReduce stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + ReductionOp reduction_op, + T initial_value, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Reduce(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_segments, + d_begin_offsets, + d_end_offsets, + reduction_op, + initial_value, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Sum(d_temp_storage, temp_storage_bytes, @@ -90,15 +119,37 @@ struct DeviceSegmentedReduce } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Sum(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Min(d_temp_storage, temp_storage_bytes, @@ -111,15 +162,37 @@ struct DeviceSegmentedReduce } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Min(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, @@ -132,15 +205,37 @@ struct DeviceSegmentedReduce } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ArgMin(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Max(d_temp_storage, temp_storage_bytes, @@ -153,15 +248,37 @@ struct DeviceSegmentedReduce } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Max(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, @@ -172,6 +289,29 @@ struct DeviceSegmentedReduce d_end_offsets, stream)); } + + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return ArgMax(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp index 5b193d15b7c7..b31bada2f27d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -35,24 +35,20 @@ #include // IWYU pragma: export -#include // IWYU pragma: export -using ::cuda::std::int64_t; - BEGIN_HIPCUB_NAMESPACE struct DeviceSegmentedSort { template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -66,16 +62,41 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeys(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortKeysDescending(d_temp_storage, @@ -90,15 +111,39 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -111,15 +156,38 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeys(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortKeysDescending(d_temp_storage, @@ -133,16 +201,38 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::StableSortKeys(d_temp_storage, temp_storage_bytes, @@ -156,16 +246,41 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortKeys(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortKeysDescending(d_temp_storage, @@ -180,15 +295,39 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::StableSortKeys(d_temp_storage, temp_storage_bytes, @@ -201,15 +340,38 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortKeys(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortKeysDescending(d_temp_storage, @@ -222,22 +384,44 @@ struct DeviceSegmentedSort stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortKeysDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, @@ -256,18 +440,50 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairs(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortPairsDescending(d_temp_storage, @@ -287,16 +503,47 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, @@ -313,16 +560,44 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairs(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortPairsDescending(d_temp_storage, @@ -340,18 +615,45 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return SortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairs(d_temp_storage, @@ -371,18 +673,50 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortPairs(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairsDescending(d_temp_storage, @@ -402,16 +736,47 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys_in, + d_keys_out, + d_values_in, + d_values_out, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairs(d_temp_storage, @@ -429,16 +794,44 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortPairs(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } + + template + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairsDescending(d_temp_storage, @@ -451,6 +844,34 @@ struct DeviceSegmentedSort d_end_offsets, stream)); } + + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return StableSortPairsDescending(d_temp_storage, + temp_storage_bytes, + d_keys, + d_values, + num_items, + num_segments, + d_begin_offsets, + d_end_offsets, + stream); + } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp index 9e8f76f1a730..6812c5cfeb72 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -66,6 +66,32 @@ class DeviceSelect stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t Flagged(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + FlagIterator d_flags, + OutputIteratorT d_out, + NumSelectedIteratorT d_num_selected_out, + int64_t num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Flagged(d_temp_storage, + temp_storage_bytes, + d_in, + d_flags, + d_out, + d_num_selected_out, + num_items, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t Flagged(void* d_temp_storage, @@ -86,6 +112,27 @@ class DeviceSelect stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t Flagged(void* d_temp_storage, + size_t& temp_storage_bytes, + IteratorT d_data, + FlagIterator d_flags, + NumSelectedIteratorT d_num_selected_out, + int64_t num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Flagged(d_temp_storage, + temp_storage_bytes, + d_data, + d_flags, + d_num_selected_out, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t If(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumSelectedIteratorT d_num_selected_out, + int64_t num_items, + SelectOp select_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return If(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + d_num_selected_out, + num_items, + select_op, + stream); + } + template HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, @@ -129,6 +202,27 @@ class DeviceSelect stream)); } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t If(void* d_temp_storage, + size_t& temp_storage_bytes, + IteratorT d_data, + NumSelectedIteratorT d_num_selected_out, + int64_t num_items, + SelectOp select_op, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return If(d_temp_storage, + temp_storage_bytes, + d_data, + d_num_selected_out, + num_items, + select_op, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t Unique(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + NumSelectedIteratorT d_num_selected_out, + int64_t num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + return Unique(d_temp_storage, + temp_storage_bytes, + d_in, + d_out, + d_num_selected_out, + num_items, + stream); + } + template + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t UniqueByKey(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys_input, + ValueIteratorT d_values_input, + OutputKeyIteratorT d_keys_output, + OutputValueIteratorT d_values_output, + NumSelectedIteratorT d_num_selected_out, + NumItemsT num_items, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return UniqueByKey(d_temp_storage, temp_storage_bytes, d_keys_input, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_spmv.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_spmv.hpp new file mode 100644 index 000000000000..dbbd5e0cb10c --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_spmv.hpp @@ -0,0 +1,140 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_CUB_DEVICE_DEVICE_SPMV_HPP_ +#define HIPCUB_CUB_DEVICE_DEVICE_SPMV_HPP_ + +#include "../../../config.hpp" +#include "../../../util_deprecated.hpp" + +#include // IWYU pragma: export +#include // IWYU pragma: export + +BEGIN_HIPCUB_NAMESPACE + +class HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") DeviceSpmv +{ + +public: + template ///< Signed integer type for sequence offsets + struct HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") SpmvParams + { + ValueT* + d_values; ///< Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix A. + OffsetT* + d_row_end_offsets; ///< Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values + OffsetT* + d_column_indices; ///< Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix A. (Indices are zero-valued.) + ValueT* + d_vector_x; ///< Pointer to the array of \p num_cols values corresponding to the dense input vector x + ValueT* + d_vector_y; ///< Pointer to the array of \p num_rows values corresponding to the dense output vector y + int num_rows; ///< Number of rows of matrix A. + int num_cols; ///< Number of columns of matrix A. + int num_nonzeros; ///< Number of nonzero elements of matrix A. + ValueT alpha; ///< Alpha multiplicand + ValueT beta; ///< Beta addend-multiplicand + + ::cub::TexObjInputIterator t_vector_x; + }; + + template + HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") + HIPCUB_RUNTIME_FUNCTION static hipError_t CsrMV(void* d_temp_storage, + size_t& temp_storage_bytes, + ValueT* d_values, + int* d_row_offsets, + int* d_column_indices, + ValueT* d_vector_x, + ValueT* d_vector_y, + int num_rows, + int num_cols, + int num_nonzeros, + hipStream_t stream = 0) + { + _CCCL_SUPPRESS_DEPRECATED_PUSH + ::cub::SpmvParams spmv_params; + _CCCL_SUPPRESS_DEPRECATED_POP + spmv_params.d_values = d_values; + spmv_params.d_row_end_offsets = d_row_offsets + 1; + spmv_params.d_column_indices = d_column_indices; + spmv_params.d_vector_x = d_vector_x; + spmv_params.d_vector_y = d_vector_y; + spmv_params.num_rows = num_rows; + spmv_params.num_cols = num_cols; + spmv_params.num_nonzeros = num_nonzeros; + spmv_params.alpha = 1.0; + spmv_params.beta = 0.0; + + _CCCL_SUPPRESS_DEPRECATED_PUSH + return static_cast( + ::cub::DispatchSpmv::Dispatch(d_temp_storage, + temp_storage_bytes, + spmv_params, + stream)); + _CCCL_SUPPRESS_DEPRECATED_POP + } + + template + HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + CsrMV(void* d_temp_storage, + size_t& temp_storage_bytes, + ValueT* d_values, + int* d_row_offsets, + int* d_column_indices, + ValueT* d_vector_x, + ValueT* d_vector_y, + int num_rows, + int num_cols, + int num_nonzeros, + hipStream_t stream, + bool debug_synchronous) + { + HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); + _CCCL_SUPPRESS_DEPRECATED_PUSH + return CsrMV(d_temp_storage, + temp_storage_bytes, + d_values, + d_row_offsets, + d_column_indices, + d_vector_x, + d_vector_y, + num_rows, + num_cols, + num_nonzeros, + stream); + _CCCL_SUPPRESS_DEPRECATED_POP + } +}; + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_CUB_DEVICE_DEVICE_SELECT_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp index 299d843a7bda..2f8157c7bb46 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -67,6 +67,7 @@ #include "device/device_segmented_reduce.hpp" #include "device/device_segmented_sort.hpp" #include "device/device_select.hpp" +#include "device/device_spmv.hpp" #include "device/device_transform.hpp" // Grid diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp index 0918d3ff0498..ab0a1f7630ca 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,8 +27,6 @@ #include -#include - BEGIN_HIPCUB_NAMESPACE namespace detail @@ -39,111 +37,6 @@ using accumulator_t = ::cuda::std::__accumulator_t; } // namespace detail -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::equal_to instead.") Equality -{ - template - HIPCUB_HOST_DEVICE - inline constexpr bool operator()(T&& t, U&& u) const - { - return ::cuda::std::forward(t) == ::cuda::std::forward(u); - } -}; - -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") Inequality -{ - template - HIPCUB_HOST_DEVICE - inline constexpr bool operator()(T&& t, U&& u) const - { - return ::cuda::std::forward(t) != ::cuda::std::forward(u); - } -}; - -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::plus instead.") Sum -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) - { - return ::cuda::std::forward(t) + ::cuda::std::forward(u); - } -}; - -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::minus instead.") Difference -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) - { - return ::cuda::std::forward(t) - ::cuda::std::forward(u); - } -}; - -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::divides instead") Division -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) - { - return std::forward(t) / std::forward(u); - } -}; - -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::maximum instead.") Max -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& t, const U& u) const -> - typename ::cuda::std::common_type::type - { - using R = typename ::cuda::std::common_type::type; - return (t < u) ? static_cast(u) : static_cast(t); - } -}; - -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::minimum instead") Min -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& t, const U& u) const -> - typename ::cuda::std::common_type::type - { - using R = typename ::cuda::std::common_type::type; - return (u < t) ? static_cast(u) : static_cast(t); - } -}; - -struct ArgMax -{ - template - HIPCUB_HOST_DEVICE - inline constexpr ::cub::KeyValuePair - operator()(const ::cub::KeyValuePair& a, - const ::cub::KeyValuePair& b) const - { - return ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a; - } -}; - -struct ArgMin -{ - template - HIPCUB_HOST_DEVICE - inline constexpr ::cub::KeyValuePair - operator()(const ::cub::KeyValuePair& a, - const ::cub::KeyValuePair& b) const - { - return ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a; - } -}; - END_HIPCUB_NAMESPACE #endif // HIPCUB_CUB_THREAD_THREAD_OPERATORS_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_store.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_store.hpp deleted file mode 100644 index 02a260e5c5f5..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_store.hpp +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -#ifndef HIPCUB_BACKEND_CUB_THREAD_STORE_HPP_ -#define HIPCUB_BACKEND_CUB_THREAD_STORE_HPP_ - -#include "../../../config.hpp" -#include "../util_type.hpp" - -#include // CUB thread store - -#include -#include - -BEGIN_HIPCUB_NAMESPACE - -enum CacheStoreModifier -{ - STORE_DEFAULT = 0, - STORE_WB = 1, - STORE_CG = 2, - STORE_CS = 3, - STORE_WT = 4, - STORE_VOLATILE = 5 -}; - -template -struct cub_cache_store_modifier_map -{ - static constexpr ::cub::CacheStoreModifier value = static_cast<::cub::CacheStoreModifier>(MOD); -}; - -template -HIPCUB_DEVICE -HIPCUB_FORCEINLINE void ThreadStoreVolatilePtr(T* ptr, T val, Fundamental /*is_fundamental*/) -{ - ::cub::ThreadStore<::cub::STORE_VOLATILE>(ptr, val); -} - -template -HIPCUB_DEVICE -HIPCUB_FORCEINLINE void ThreadStore(T* ptr, - T val, - ::std::integral_constant /*modifier*/, - ::std::true_type /*is_pointer*/) -{ - ::cub::ThreadStore::value>(ptr, val); -} - -template -HIPCUB_DEVICE -HIPCUB_FORCEINLINE void ThreadStore(OutputIteratorT itr, - T val, - ::std::integral_constant /*modifier*/, - ::std::false_type /*is_pointer*/) -{ - ThreadStore(&(*itr), - val, - ::std::integral_constant{}, - ::std::true_type{}); -} - -template -HIPCUB_DEVICE -HIPCUB_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val) -{ - ThreadStore(itr, - val, - ::std::integral_constant{}, - ::std::bool_constant<_HIPCUB_STD::is_pointer::value>()); -} - -namespace detail -{ - -template -struct iterate_thread_store -{ - template - static HIPCUB_DEVICE - HIPCUB_FORCEINLINE void Store(T* ptr, T* vals) - { - ThreadStore(ptr + COUNT, - vals[COUNT], - ::std::integral_constant{}, - ::std::true_type{}); - iterate_thread_store::template Store(ptr, vals); - } - - template - static HIPCUB_DEVICE - HIPCUB_FORCEINLINE void Dereference(OutputIteratorT ptr, T* vals) - { - ptr[COUNT] = vals[COUNT]; - iterate_thread_store::Dereference(ptr, vals); - } -}; - -template -struct iterate_thread_store -{ - template - static HIPCUB_DEVICE - HIPCUB_FORCEINLINE void Store(T* /*ptr*/, T* /*vals*/) - {} - - template - static HIPCUB_DEVICE - HIPCUB_FORCEINLINE void Dereference(OutputIteratorT /*ptr*/, T* /*vals*/) - {} -}; - -} // namespace detail - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_BACKEND_CUB_THREAD_STORE_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp index 2489e3af162f..794d527aade1 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,4 +34,44 @@ #include // IWYU pragma: export +BEGIN_HIPCUB_NAMESPACE + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_MAX + /// Select maximum(a, b) + #define HIPCUB_MAX(a, b) (((b) > (a)) ? (b) : (a)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_MIN + /// Select minimum(a, b) + #define HIPCUB_MIN(a, b) (((b) < (a)) ? (b) : (a)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_QUOTIENT_FLOOR + /// Quotient of x/y rounded down to nearest integer + #define HIPCUB_QUOTIENT_FLOOR(x, y) ((x) / (y)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_QUOTIENT_CEILING + /// Quotient of x/y rounded up to nearest integer + #define HIPCUB_QUOTIENT_CEILING(x, y) (((x) + (y)-1) / (y)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_ROUND_UP_NEAREST + /// x rounded up to the nearest multiple of y + #define HIPCUB_ROUND_UP_NEAREST(x, y) (HIPCUB_QUOTIENT_CEILING(x, y) * y) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_ROUND_DOWN_NEAREST + /// x rounded down to the nearest multiple of y + #define HIPCUB_ROUND_DOWN_NEAREST(x, y) (((x) / (y)) * y) +#endif + +END_HIPCUB_NAMESPACE + #endif // HIPCUB_CUB_MACRO_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp index daf1e265b1d1..fc67d645b142 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,4 +34,39 @@ #include // IWYU pragma: export +BEGIN_HIPCUB_NAMESPACE + +/// \brief Alias temporaries to externally-allocated device storage (or simply return the amount of storage needed). +/// \tparam ALLOCATIONS The number of allocations that are needed. +/// \param d_temp_storage [in] Device-accessible allocation of temporary storage. When nullptr, the required allocation size is written to \p temp_storage_bytes and no work is done. +/// \param temp_storage_bytes [in,out] Size in bytes of \t d_temp_storage allocation. +/// \param allocations [out] Pointers to device allocations needed. +/// \param allocation_sizes [in] Sizes in bytes of device allocations needed. +template +HIPCUB_DEPRECATED_BECAUSE("Internal-only implementation detail") +HIPCUB_HOST_DEVICE HIPCUB_FORCEINLINE hipError_t + AliasTemporaries(void* d_temp_storage, + size_t& temp_storage_bytes, + void* (&allocations)[ALLOCATIONS], + const size_t (&allocation_sizes)[ALLOCATIONS]) +{ + cudaError_t error = ::cub::detail::AliasTemporaries(d_temp_storage, + temp_storage_bytes, + allocations, + allocation_sizes); + + if(cudaSuccess == error) + { + return hipSuccess; + } + else if(cudaErrorInvalidValue == error) + { + return hipErrorInvalidValue; + } + + return hipErrorUnknown; +} + +END_HIPCUB_NAMESPACE + #endif // HIPCUB_CUB_UTIL_TEMPORARY_STORAGE_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_type.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_type.hpp deleted file mode 100644 index 3ec0ebd64dbc..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_type.hpp +++ /dev/null @@ -1,93 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_CUB_UTIL_TYPE_HPP_ -#define HIPCUB_CUB_UTIL_TYPE_HPP_ - -#include "../../config.hpp" -#include "../../util_deprecated.hpp" - -#include _HIPCUB_STD_INCLUDE(iterator) -#include _HIPCUB_STD_INCLUDE(type_traits) - -#include // IWYU pragma: export - -BEGIN_HIPCUB_NAMESPACE - -namespace detail -{ -// the following iterator helpers are not named iter_value_t etc, like the C++20 facilities, because they are defined in -// terms of C++17 iterator_traits and not the new C++20 indirectly_readable trait etc. This allows them to detect nested -// value_type, difference_type and reference aliases, which the new C+20 traits do not consider (they only consider -// specializations of iterator_traits). Also, a value_type of void remains supported (needed by some output iterators). - -template -struct it_traits -{ - using value_type = typename _HIPCUB_STD::iterator_traits::value_type; - using reference = typename _HIPCUB_STD::iterator_traits::reference; - using difference_type = typename _HIPCUB_STD::iterator_traits::difference_type; - using pointer = typename _HIPCUB_STD::iterator_traits::pointer; -}; -template -struct it_traits> -{ - using value_type = typename It::value_type; - using reference = typename It::reference; - using difference_type = typename It::difference_type; - using pointer = typename It::pointer; -}; -template -using it_value_t = typename it_traits::value_type; -template -using it_reference_t = typename it_traits::reference; -template -using it_difference_t = typename it_traits::difference_type; -template -using it_pointer_t = typename it_traits::pointer; - -// use this whenever you need to lazily evaluate a trait. E.g., as an alternative in replace_if_use_default. -template typename Trait, typename... Args> -struct lazy_trait -{ - using type = Trait; -}; - -template -using int_constant_t = _HIPCUB_STD::integral_constant; - -} // namespace detail - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_CUB_UTIL_TYPE_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp index 59060b602f8e..837ef4b1a98c 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -336,11 +336,13 @@ class ScanTileState * \tparam T Type of the values scanned. * \tparam ScanOpT Scan operation type. * \tparam ScanTileStateT Scan status type. + * \tparam LEGACY_PTX_ARCH [optional] Unused (deprecated). * \tparam DelayConstructorT [optional] Unused (CUB's implementation detail). */ template */> class TilePrefixCallbackOp { diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp index 29d29dee9391..d306c834e3aa 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. -* Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. +* Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,7 +32,6 @@ #include "../../../config.hpp" #include "../thread/thread_sort.hpp" -#include "../util_macro.hpp" #include "../util_math.hpp" #include "../util_type.hpp" @@ -96,7 +95,7 @@ HIPCUB_DEVICE __forceinline__ void SerialMerge(KeyT *keys_shared, KeyT key1 = keys_shared[keys1_beg]; KeyT key2 = keys_shared[keys2_beg]; - _CCCL_SORT_MAYBE_UNROLL() +#pragma unroll for (int item = 0; item < ITEMS_PER_THREAD; ++item) { bool p = (keys2_beg < keys2_end) && @@ -388,7 +387,7 @@ class BlockMergeSortStrategy // KeyT max_key = oob_default; - _CCCL_SORT_MAYBE_UNROLL() + #pragma unroll for (int item = WARP_SORT ? 1 : 0; item < ITEMS_PER_THREAD; ++item) { if (ITEMS_PER_THREAD * static_cast(linear_tid) + item < valid_items) @@ -412,7 +411,7 @@ class BlockMergeSortStrategy // each thread has sorted keys // merge sort keys in shared memory // - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (int target_merged_threads_number = 2; target_merged_threads_number <= NUM_THREADS; target_merged_threads_number *= 2) @@ -424,7 +423,7 @@ class BlockMergeSortStrategy // store keys in shmem // - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (int item = 0; item < ITEMS_PER_THREAD; ++item) { int idx = ITEMS_PER_THREAD * linear_tid + item; @@ -483,7 +482,7 @@ class BlockMergeSortStrategy // store keys in shmem // - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (int item = 0; item < ITEMS_PER_THREAD; ++item) { int idx = ITEMS_PER_THREAD * linear_tid + item; @@ -494,7 +493,7 @@ class BlockMergeSortStrategy // gather items from shmem // - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (int item = 0; item < ITEMS_PER_THREAD; ++item) { items[item] = temp_storage.items_shared[indices[item]]; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp index b4ad51da814d..598f39a6ebee 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -74,7 +74,7 @@ constexpr auto tuple_bit_size_impl() template struct tuple_bit_size<::hipcub::tuple> - : public std::integral_constant, 0>()> + : public std::integral_constant, 0>()> {}; } // namespace detail diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp index 39fe245f8b9c..8296c3a37a1b 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -37,8 +37,7 @@ #include "../util_type.hpp" #include "block_scan.hpp" -#include _HIPCUB_STD_INCLUDE(limits) - +#include #include BEGIN_HIPCUB_NAMESPACE @@ -248,7 +247,7 @@ class BlockRunLengthDecode { OffsetT lower_bound = 0; OffsetT upper_bound = num_items; - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (int i = 0; i <= Log2::VALUE; i++) { OffsetT mid = hipcub::MidPoint(lower_bound, upper_bound); @@ -273,7 +272,7 @@ class BlockRunLengthDecode { // Keep the runs' items and the offsets of each run's beginning in the temporary storage RunOffsetT thread_dst_offset = static_cast(linear_tid) * static_cast(RUNS_PER_THREAD); - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (int i = 0; i < RUNS_PER_THREAD; i++) { temp_storage.runs.run_values[thread_dst_offset] = run_values[i]; @@ -292,7 +291,7 @@ class BlockRunLengthDecode { // Compute the offset for the beginning of each run DecodedOffsetT run_offsets[RUNS_PER_THREAD]; - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (int i = 0; i < RUNS_PER_THREAD; i++) { run_offsets[i] = static_cast(run_lengths[i]); @@ -348,7 +347,7 @@ class BlockRunLengthDecode ItemT val = temp_storage.runs.run_values[assigned_run]; - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (DecodedOffsetT i = 0; i < DECODED_ITEMS_PER_THREAD; i++) { decoded_items[i] = val; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp index a4f406252298..3111e8224f2d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,9 +32,9 @@ #include "../../../config.hpp" -#include // IWYU pragma: export +#include "../thread/thread_operators.hpp" -#include _HIPCUB_STD_INCLUDE(functional) +#include // IWYU pragma: export #include @@ -121,14 +121,12 @@ class BlockScan } template - HIPCUB_DEVICE - inline void InclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE inline + void InclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::inclusive_scan(input, - output, - temp_storage_, - block_prefix_callback_op, - _HIPCUB_STD::plus<>{}); + base_type::inclusive_scan( + input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() + ); } template @@ -147,16 +145,13 @@ class BlockScan } template - HIPCUB_DEVICE - inline void InclusiveSum(T (&input)[ITEMS_PER_THREAD], - T (&output)[ITEMS_PER_THREAD], - BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE inline + void InclusiveSum(T(&input)[ITEMS_PER_THREAD], T(&output)[ITEMS_PER_THREAD], + BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::inclusive_scan(input, - output, - temp_storage_, - block_prefix_callback_op, - _HIPCUB_STD::plus<>{}); + base_type::inclusive_scan( + input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() + ); } template @@ -246,14 +241,12 @@ class BlockScan } template - HIPCUB_DEVICE - inline void ExclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE inline + void ExclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::exclusive_scan(input, - output, - temp_storage_, - block_prefix_callback_op, - _HIPCUB_STD::plus<>{}); + base_type::exclusive_scan( + input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() + ); } template @@ -272,16 +265,13 @@ class BlockScan } template - HIPCUB_DEVICE - inline void ExclusiveSum(T (&input)[ITEMS_PER_THREAD], - T (&output)[ITEMS_PER_THREAD], - BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE inline + void ExclusiveSum(T(&input)[ITEMS_PER_THREAD], T(&output)[ITEMS_PER_THREAD], + BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::exclusive_scan(input, - output, - temp_storage_, - block_prefix_callback_op, - _HIPCUB_STD::plus<>{}); + base_type::exclusive_scan( + input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() + ); } template diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp index 2c591898dd0a..8698d5767540 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,8 @@ #include "../../../config.hpp" +#include "../thread/thread_operators.hpp" + #include // IWYU pragma: export #include diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp index 4cb0033ff10e..0ead22896253 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -36,17 +36,12 @@ #define HIPCUB_ROCPRIM_BLOCK_RADIX_RANK_SORT_OPERATIONS_HPP_ #include "../../../config.hpp" -#include "../../../libcxx.hpp" #include "../util_type.hpp" #include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export -#include _HIPCUB_LIBCXX_INCLUDE(bit) - -#include - BEGIN_HIPCUB_NAMESPACE /** \brief Twiddling keys for radix sort. */ @@ -97,9 +92,8 @@ struct RadixSortTwiddle enum { - FLOAT_KEY = _HIPCUB_STD::is_floating_point_v - || std::is_same_v - || std::is_same_v, + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH FLOAT_KEY = TraitsT::CATEGORY == FLOATING_POINT, + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP }; static __device__ __forceinline__ UnsignedBits ProcessFloatMinusZero(UnsignedBits key) @@ -130,19 +124,7 @@ struct RadixSortTwiddle __device__ __forceinline__ uint32_t Digit(UnsignedBits key) { - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - - uint32_t result = -#if _HIPCUB_HAS_DEVICE_SYSTEM_STD - _HIPCUB_LIBCXX::bitfield_extract -#else - BFE -#endif - (this->ProcessFloatMinusZero(key), bit_start, num_bits); - - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - - return result; + return BFE(this->ProcessFloatMinusZero(key), bit_start, num_bits); } }; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp index 5bf19740a626..e66f8b4bf9c6 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2022-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2022-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,26 +32,24 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" +#include #include // IWYU pragma: export -#include _HIPCUB_STD_INCLUDE(functional) - BEGIN_HIPCUB_NAMESPACE struct DeviceAdjacentDifference { template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractLeftCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeftCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference(d_temp_storage, temp_storage_bytes, @@ -65,17 +63,17 @@ struct DeviceAdjacentDifference template, - typename NumItemsT = uint32_t> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractLeftCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractLeftCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractLeftCopy(d_temp_storage, @@ -88,15 +86,14 @@ struct DeviceAdjacentDifference } template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractLeft(void* d_temp_storage, - size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeft(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference_inplace(d_temp_storage, temp_storage_bytes, @@ -108,16 +105,16 @@ struct DeviceAdjacentDifference } template, - typename NumItemsT = uint32_t> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractLeft(void* d_temp_storage, - size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractLeft(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractLeft(d_temp_storage, @@ -130,16 +127,15 @@ struct DeviceAdjacentDifference template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractRightCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRightCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference_right(d_temp_storage, temp_storage_bytes, @@ -153,17 +149,17 @@ struct DeviceAdjacentDifference template, - typename NumItemsT = uint32_t> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractRightCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractRightCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractRightCopy(d_temp_storage, @@ -176,15 +172,14 @@ struct DeviceAdjacentDifference } template, - typename NumItemsT = uint32_t> - static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractRight(void* d_temp_storage, - size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRight(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference_right_inplace(d_temp_storage, temp_storage_bytes, @@ -196,16 +191,16 @@ struct DeviceAdjacentDifference } template, - typename NumItemsT = uint32_t> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION - hipError_t SubtractRight(void* d_temp_storage, - size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = ::hipcub::Difference, + typename NumItemsT = std::uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t + SubtractRight(void* d_temp_storage, + std::size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractRight(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp index 68f2f2744ec6..8e41fa0fd330 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,13 +40,13 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceCopy { template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - _HIPCUB_STD::int64_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + uint32_t num_buffers, + hipStream_t stream = 0) { return rocprim::batch_copy(d_temp_storage, temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp index 32a672b5d4c7..682eb673a388 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,11 +31,12 @@ #include "../../../config.hpp" +#include "../iterator/counting_input_iterator.hpp" +#include "../iterator/discard_output_iterator.hpp" +#include "../thread/thread_operators.hpp" #include "../util_mdspan.hpp" #include // IWYU pragma: export -#include // IWYU pragma: export -#include // IWYU pragma: export #include @@ -116,14 +117,15 @@ struct DeviceFor OpT op, hipStream_t stream = 0) -> std::enable_if_t()), - detail::it_value_t>::value, + typename std::iterator_traits< + RandomAccessIteratorT>::value_type>::value, hipError_t> { - using T = detail::it_value_t; - + using T = typename std::iterator_traits::value_type; + using OutputIterator = typename rocprim::discard_iterator; detail::bulk::OpWrapper wrapper_op = {op}; - auto output = rocprim::make_discard_iterator(); + OutputIterator output; return rocprim::transform(first, output, @@ -135,15 +137,14 @@ struct DeviceFor template HIPCUB_RUNTIME_FUNCTION - static auto ForEachN(RandomAccessIteratorT first, - OffsetT num_items, - OpT op, - hipStream_t stream = 0) - -> std::enable_if_t()), - detail::it_value_t>::value, - hipError_t> + static auto + ForEachN(RandomAccessIteratorT first, OffsetT num_items, OpT op, hipStream_t stream = 0) + -> std::enable_if_t()), + typename std::iterator_traits< + RandomAccessIteratorT>::value_type>::value, + hipError_t> { - using T = detail::it_value_t; + using T = typename std::iterator_traits::value_type; detail::bulk::OpWrapper wrapper_op = {op}; @@ -206,7 +207,7 @@ HIPCUB_RUNTIME_FUNCTION OpT op, hipStream_t stream = 0) { - using offset_t = detail::it_difference_t; + using offset_t = typename std::iterator_traits::difference_type; const offset_t num_items = static_cast(std::distance(first, last)); return ForEachN(first, num_items, op, stream); @@ -264,12 +265,11 @@ HIPCUB_RUNTIME_FUNCTION { static_assert(std::is_integral::value, "ShapeT must be an integral type"); using InputIterator = typename rocprim::counting_iterator; - + using OutputIterator = typename rocprim::discard_iterator; detail::bulk::OpWrapper wrapper_op = {op}; InputIterator input(ShapeT(0)); - - auto output = rocprim::make_discard_iterator(); + OutputIterator output; return rocprim::transform(input, output, @@ -350,13 +350,14 @@ HIPCUB_RUNTIME_FUNCTION // rocprim::counting_iterator only holds the index, not the data. using InputIterator = typename rocprim::counting_iterator; + // We don't actually need the output, so we use rocprim::discard_iterator here as a placeholder. + using OutputIterator = typename rocprim::discard_iterator; // How many times rocprim::transform will iterate. constexpr auto ext_size = ::hipcub::extents_size::value; InputIterator input(IndexType(0)); // Initialize the input iterator, starting from 0. - - auto output = rocprim::make_discard_iterator(); + OutputIterator output; // `ForEachInExtents` only iterates over the extents on device and does not guarantee ordering. // We only need to invoke `$op` `$ext_size` times. Therefore, `rocprim::transform` is suitable. diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp index ab51c2e02acd..0dd5d978cb34 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -61,25 +61,24 @@ HIPCUB_FORCEINLINE bool may_overflow(LevelT lower_level, ::std::true_type /* is_integral */) { return static_cast(upper_level - lower_level) - > (_HIPCUB_STD::numeric_limits::max() - / static_cast(num_bins)); + > (::std::numeric_limits::max() / static_cast(num_bins)); } template struct int_arithmetic_t { - using type - = ::std::conditional_t - || ::std::is_same_v), - CommonT, - uint64_t> + using type = ::std::conditional_t< + sizeof(SampleT) + sizeof(CommonT) <= sizeof(uint32_t), + uint32_t, +#if HIPCUB_IS_INT128_ENABLED + ::std::conditional_t<(::std::is_same::value + || ::std::is_same::value), + CommonT, + uint64_t> #else - uint64_t + uint64_t #endif - >; + >; }; // If potential overflow is detected, returns hipErrorInvalidValue, otherwise hipSuccess. @@ -87,7 +86,7 @@ template HIPCUB_HOST_DEVICE HIPCUB_FORCEINLINE hipError_t check_overflow(LevelT lower_level, LevelT upper_level, int num_levels) { - using sample_type = it_value_t; + using sample_type = typename std::iterator_traits::value_type; using common_type = typename hipcub::common_type::type; static_assert(std::is_convertible::value, "The common type of `LevelT` and `SampleT` must be " diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp index 36dac611cff8..a5828b7ad4d5 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2023-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2023-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,20 +40,14 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceMemcpy { template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - _HIPCUB_STD::int64_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + uint32_t num_buffers, + hipStream_t stream = 0) { - if(num_buffers == 0) - { - temp_storage_bytes = 0; - return hipSuccess; - } - return rocprim::batch_memcpy(d_temp_storage, temp_storage_bytes, input_buffer_it, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp index dc80f72a2491..014695e5ba01 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp @@ -1,6 +1,6 @@ /****************************************************************************** - * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2025-2026, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -43,15 +43,15 @@ struct DeviceMerge typename KeyIteratorOut, typename CompareOp = ::rocprim::less<>> HIPCUB_RUNTIME_FUNCTION - static hipError_t MergeKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorIn1 keys_in1, - _HIPCUB_STD::int64_t num_keys1, - KeyIteratorIn2 keys_in2, - _HIPCUB_STD::int64_t num_keys2, - KeyIteratorOut keys_out, - CompareOp compare_op = {}, - hipStream_t stream = 0) + static hipError_t MergeKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorIn1 keys_in1, + int num_keys1, + KeyIteratorIn2 keys_in2, + int num_keys2, + KeyIteratorOut keys_out, + CompareOp compare_op = {}, + hipStream_t stream = 0) { return ::rocprim::merge(d_temp_storage, @@ -74,18 +74,18 @@ struct DeviceMerge typename ValueIteratorOut, typename CompareOp = ::rocprim::less<>> HIPCUB_RUNTIME_FUNCTION - static hipError_t MergePairs(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorIn1 keys_in1, - ValueIteratorIn1 values_in1, - _HIPCUB_STD::int64_t num_keys1, - KeyIteratorIn2 keys_in2, - ValueIteratorIn2 values_in2, - _HIPCUB_STD::int64_t num_keys2, - KeyIteratorOut keys_out, - ValueIteratorOut values_out, - CompareOp compare_op = {}, - hipStream_t stream = 0) + static hipError_t MergePairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorIn1 keys_in1, + ValueIteratorIn1 values_in1, + int num_keys1, + KeyIteratorIn2 keys_in2, + ValueIteratorIn2 values_in2, + int num_keys2, + KeyIteratorOut keys_out, + ValueIteratorOut values_out, + CompareOp compare_op = {}, + hipStream_t stream = 0) { return ::rocprim::merge(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp index 5bf1e08e9e96..0c9f4df4d82a 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,14 +42,13 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceMergeSort { template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -64,15 +63,15 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairs(d_temp_storage, @@ -90,16 +89,15 @@ struct DeviceMergeSort typename ValueIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - ValueInputIteratorT d_input_items, - KeyIteratorT d_output_keys, - ValueIteratorT d_output_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + ValueInputIteratorT d_input_items, + KeyIteratorT d_output_keys, + ValueIteratorT d_output_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -119,17 +117,17 @@ struct DeviceMergeSort typename ValueIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - ValueInputIteratorT d_input_items, - KeyIteratorT d_output_keys, - ValueIteratorT d_output_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + ValueInputIteratorT d_input_items, + KeyIteratorT d_output_keys, + ValueIteratorT d_output_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairsCopy(d_temp_storage, @@ -144,13 +142,12 @@ struct DeviceMergeSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -163,14 +160,14 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items, compare_op, stream); @@ -180,14 +177,13 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, @@ -204,15 +200,15 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); @@ -226,14 +222,13 @@ struct DeviceMergeSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -248,15 +243,15 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairs(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairs(d_temp_storage, @@ -269,13 +264,12 @@ struct DeviceMergeSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -288,14 +282,14 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeys(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeys(d_temp_storage, @@ -310,14 +304,13 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -333,15 +326,15 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysCopy(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysCopy(void* d_temp_storage, + std::size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysCopy(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp index cf50fe274167..55ca243d0a71 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -155,7 +155,7 @@ struct DevicePartition typename NumItemsT> HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, - size_t& temp_storage_bytes, + std::size_t& temp_storage_bytes, InputIteratorT d_in, FirstOutputIteratorT d_first_part_out, SecondOutputIteratorT d_second_part_out, @@ -190,7 +190,7 @@ struct DevicePartition typename NumItemsT> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, - size_t& temp_storage_bytes, + std::size_t& temp_storage_bytes, InputIteratorT d_in, FirstOutputIteratorT d_first_part_out, SecondOutputIteratorT d_second_part_out, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp index 41b697eb3a52..8f21617f81ec 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -46,11 +46,8 @@ #include // hip_bfloat16 #include // __half -#include _HIPCUB_LIBCXX_INCLUDE(functional) -#include _HIPCUB_STD_INCLUDE(functional) -#include _HIPCUB_STD_INCLUDE(limits) - #include +#include BEGIN_HIPCUB_NAMESPACE namespace detail @@ -69,7 +66,7 @@ HIPCUB_HOST_DEVICE T set_half_bits(uint16_t value) template HIPCUB_HOST_DEVICE inline T get_lowest_value() { - return _HIPCUB_STD::numeric_limits::lowest(); + return std::numeric_limits::lowest(); } template<> @@ -89,7 +86,7 @@ HIPCUB_HOST_DEVICE inline hip_bfloat16 get_lowest_value() template HIPCUB_HOST_DEVICE inline T get_max_value() { - return _HIPCUB_STD::numeric_limits::max(); + return std::numeric_limits::max(); } template<> @@ -119,7 +116,7 @@ template inline auto get_lowest_special_value() -> typename std::enable_if_t::value, T> { - return -_HIPCUB_STD::numeric_limits::infinity(); + return -std::numeric_limits::infinity(); } template<> @@ -149,7 +146,7 @@ template inline auto get_max_special_value() -> typename std::enable_if_t::value, T> { - return _HIPCUB_STD::numeric_limits::infinity(); + return std::numeric_limits::infinity(); } template<> @@ -232,15 +229,15 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using InputT = detail::it_value_t; - using OutputT = detail::it_value_t; - using InitT = detail::non_void_value_t; + using InputT = typename std::iterator_traits::value_type; + using OutputT = typename std::iterator_traits::value_type; + using InitT = hipcub::detail::non_void_value_t; return Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, - _HIPCUB_STD::plus<>{}, + ::hipcub::Sum(), InitT(0), stream); } @@ -267,17 +264,13 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using T = detail::it_value_t; + using T = typename std::iterator_traits::value_type; return Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, -#if _HIPCUB_HAS_DEVICE_SYSTEM_STD - _HIPCUB_LIBCXX::minimum<>{}, -#else - [] (auto a, auto b) { return a > b ? b : a;}, -#endif + ::hipcub::Min(), detail::get_max_value(), stream); } @@ -309,8 +302,8 @@ class DeviceReduce hipStream_t stream = 0) { using OffsetT = NumItemsT; - using T = detail::it_value_t; - using O = detail::it_value_t; + using T = typename std::iterator_traits::value_type; + using O = typename std::iterator_traits::value_type; using OutputTupleT = hipcub::detail::non_void_value_t>; using OutputValueT = typename OutputTupleT::Value; @@ -388,17 +381,13 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using T = detail::it_value_t; + using T = typename std::iterator_traits::value_type; return Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, -#if _HIPCUB_HAS_DEVICE_SYSTEM_STD - _HIPCUB_LIBCXX::maximum<>{}, -#else - [] (auto a, auto b) { return a > b ? a : b;}, -#endif + ::hipcub::Max(), detail::get_lowest_value(), stream); } @@ -430,9 +419,9 @@ class DeviceReduce hipStream_t stream = 0) { using OffsetT = NumItemsT; - using T = detail::it_value_t; - using O = detail::it_value_t; - using OutputTupleT = detail::non_void_value_t>; + using T = typename std::iterator_traits::value_type; + using O = typename std::iterator_traits::value_type; + using OutputTupleT = hipcub::detail::non_void_value_t>; using OutputValueT = typename OutputTupleT::Value; using IteratorT = ArgIndexInputIterator; @@ -557,7 +546,8 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using key_compare_op = ::rocprim::equal_to>; + using key_compare_op + = ::rocprim::equal_to::value_type>; return ::rocprim::reduce_by_key(d_temp_storage, temp_storage_bytes, d_keys_in, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp index de2be6e0859d..2e8d70581a3a 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,8 +40,6 @@ #include // IWYU pragma: export #include // IWYU pragma: export -#include _HIPCUB_STD_INCLUDE(functional) - BEGIN_HIPCUB_NAMESPACE class DeviceScan @@ -60,7 +58,7 @@ class DeviceScan temp_storage_bytes, d_in, d_out, - _HIPCUB_STD::plus<>{}, + ::hipcub::Sum(), num_items, stream); } @@ -232,12 +230,12 @@ class DeviceScan NumItemsT num_items, hipStream_t stream = 0) { - using T = detail::it_value_t; + using T = typename std::iterator_traits::value_type; return ExclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, - _HIPCUB_STD::plus<>{}, + ::hipcub::Sum(), T(0), num_items, stream); @@ -494,8 +492,8 @@ class DeviceScan template, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -506,14 +504,14 @@ class DeviceScan EqualityOpT equality_op = EqualityOpT(), hipStream_t stream = 0) { - using in_value_type = detail::it_value_t; + using in_value_type = typename std::iterator_traits::value_type; return ExclusiveScanByKey(d_temp_storage, temp_storage_bytes, d_keys_in, d_values_in, d_values_out, - _HIPCUB_STD::plus<>{}, + ::hipcub::Sum(), static_cast(0), num_items, equality_op, @@ -523,8 +521,8 @@ class DeviceScan template, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -552,8 +550,8 @@ class DeviceScan typename ValuesOutputIteratorT, typename ScanOpT, typename InitValueT, - typename EqualityOpT = _HIPCUB_STD::equal_to<>, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -593,8 +591,8 @@ class DeviceScan typename ValuesOutputIteratorT, typename ScanOpT, typename InitValueT, - typename EqualityOpT = _HIPCUB_STD::equal_to<>, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -624,8 +622,8 @@ class DeviceScan template, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -641,7 +639,7 @@ class DeviceScan d_keys_in, d_values_in, d_values_out, - _HIPCUB_STD::plus<>{}, + ::hipcub::Sum(), num_items, equality_op, stream); @@ -650,8 +648,8 @@ class DeviceScan template, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -678,8 +676,8 @@ class DeviceScan typename ValuesInputIteratorT, typename ValuesOutputIteratorT, typename ScanOpT, - typename EqualityOpT = _HIPCUB_STD::equal_to<>, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -691,7 +689,8 @@ class DeviceScan EqualityOpT equality_op = EqualityOpT(), hipStream_t stream = 0) { - using acc_t = ::rocprim::accumulator_t>; + using acc_t = ::rocprim:: + accumulator_t::value_type>; return ::rocprim::inclusive_scan_by_key<::rocprim::default_config, KeysInputIteratorT, @@ -715,8 +714,8 @@ class DeviceScan typename ValuesInputIteratorT, typename ValuesOutputIteratorT, typename ScanOpT, - typename EqualityOpT = _HIPCUB_STD::equal_to<>, - typename NumItemsT = uint32_t> + typename EqualityOpT = ::hipcub::Equality, + typename NumItemsT = std::uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp index 1a05059891de..8f6494471209 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp @@ -42,12 +42,9 @@ #include // IWYU pragma: export #include // IWYU pragma: export -#include _HIPCUB_LIBCXX_INCLUDE(functional) -#include _HIPCUB_STD_INCLUDE(functional) -#include _HIPCUB_STD_INCLUDE(limits) - #include #include +#include BEGIN_HIPCUB_NAMESPACE @@ -89,7 +86,7 @@ inline hipError_t launch_segmented_arg_minmax(::rocprim::detail::target current_ const unsigned int segment_id = ::rocprim::detail::block_id<0>(); // Large indices need bigger offset type than unsigned int - using offset_type = it_value_t; + using offset_type = typename std::iterator_traits::value_type; const offset_type begin_offset = begin_offsets[segment_id]; const offset_type end_offset = end_offsets[segment_id]; @@ -136,7 +133,7 @@ inline hipError_t segmented_arg_minmax(void* temporary_storage, InitValueType empty_value, hipStream_t stream) { - using input_type = detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; using result_type = ::rocprim::accumulator_t; using selector = ::rocprim::detail::segmented_reduce_config_selector; @@ -159,7 +156,7 @@ inline hipError_t segmented_arg_minmax(void* temporary_storage, std::chrono::high_resolution_clock::time_point start; - if constexpr(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) + if HIPCUB_IF_CONSTEXPR(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) { start = std::chrono::high_resolution_clock::now(); } @@ -191,16 +188,16 @@ struct DeviceSegmentedReduce typename ReductionOp, typename T> HIPCUB_RUNTIME_FUNCTION - static hipError_t Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - ReductionOp reduction_op, - T initial_value, - hipStream_t stream = 0) + static hipError_t Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + ReductionOp reduction_op, + T initial_value, + hipStream_t stream = 0) { return ::rocprim::segmented_reduce( d_temp_storage, @@ -221,17 +218,17 @@ struct DeviceSegmentedReduce typename ReductionOp, typename T> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - ReductionOp reduction_op, - T initial_value, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + ReductionOp reduction_op, + T initial_value, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Reduce(d_temp_storage, @@ -248,16 +245,16 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { - using input_type = detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; return Reduce(d_temp_storage, temp_storage_bytes, @@ -266,22 +263,22 @@ struct DeviceSegmentedReduce num_segments, d_begin_offsets, d_end_offsets, - _HIPCUB_STD::plus<>{}, + ::hipcub::Sum(), input_type(), stream); } template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Sum(d_temp_storage, @@ -296,16 +293,16 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { - using input_type = detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; return Reduce(d_temp_storage, temp_storage_bytes, @@ -314,26 +311,22 @@ struct DeviceSegmentedReduce num_segments, d_begin_offsets, d_end_offsets, -#if _HIPCUB_HAS_DEVICE_SYSTEM_STD - _HIPCUB_LIBCXX::minimum<>{}, -#else - [] (auto a, auto b) { return a > b ? b : a;}, -#endif - _HIPCUB_STD::numeric_limits::max(), + ::hipcub::Min(), + std::numeric_limits::max(), stream); } template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Min(d_temp_storage, @@ -348,20 +341,20 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { using OffsetT = int; - using T = hipcub::detail::it_value_t; - using O = hipcub::detail::it_value_t; - using OutputTupleT = - typename std::conditional, KeyValuePair, O>::type; + using T = typename std::iterator_traits::value_type; + using O = typename std::iterator_traits::value_type; + using OutputTupleT = typename std:: + conditional::value, KeyValuePair, O>::type; using OutputValueT = typename OutputTupleT::Value; using IteratorT = ArgIndexInputIterator; @@ -369,7 +362,7 @@ struct DeviceSegmentedReduce IteratorT d_indexed_in(d_in); // true maximum value of the full range // key is ::max because ArgMin finds the lowest value that has the lowest key - const OutputTupleT init(_HIPCUB_STD::numeric_limits::max(), + const OutputTupleT init(std::numeric_limits::max(), detail::get_max_special_value()); // special value for empty segments const OutputTupleT empty_value(1, detail::get_max_value()); @@ -389,15 +382,15 @@ struct DeviceSegmentedReduce template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return ArgMin(d_temp_storage, @@ -412,16 +405,16 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { - using input_type = detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; return Reduce(d_temp_storage, temp_storage_bytes, @@ -430,26 +423,22 @@ struct DeviceSegmentedReduce num_segments, d_begin_offsets, d_end_offsets, -#if _HIPCUB_HAS_DEVICE_SYSTEM_STD - _HIPCUB_LIBCXX::maximum<>{}, -#else - [] (auto a, auto b) { return a > b ? a : b;}, -#endif - _HIPCUB_STD::numeric_limits::lowest(), + ::hipcub::Max(), + std::numeric_limits::lowest(), stream); } template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Max(d_temp_storage, @@ -464,20 +453,20 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { using OffsetT = int; - using T = hipcub::detail::it_value_t; - using O = hipcub::detail::it_value_t; - using OutputTupleT = - typename std::conditional, KeyValuePair, O>::type; + using T = typename std::iterator_traits::value_type; + using O = typename std::iterator_traits::value_type; + using OutputTupleT = typename std:: + conditional::value, KeyValuePair, O>::type; using OutputValueT = typename OutputTupleT::Value; using IteratorT = ArgIndexInputIterator; @@ -485,7 +474,7 @@ struct DeviceSegmentedReduce IteratorT d_indexed_in(d_in); // true minimum value of the full range // key is ::max because ArgMax finds the highest value that has the lowest key - const OutputTupleT init(_HIPCUB_STD::numeric_limits::max(), + const OutputTupleT init(std::numeric_limits::max(), detail::get_lowest_special_value()); // special value for empty segments const OutputTupleT empty_value(1, detail::get_lowest_value()); @@ -505,15 +494,15 @@ struct DeviceSegmentedReduce template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return ArgMax(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp index 6765a273d7de..e266bd635aa0 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,18 +42,17 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceSegmentedSort { template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_pairs(d_temp_storage, temp_storage_bytes, @@ -72,19 +71,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairs(d_temp_storage, @@ -101,16 +100,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); ::rocprim::double_buffer d_values_db = detail::to_double_buffer(d_values); @@ -132,17 +130,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairs(d_temp_storage, @@ -157,18 +155,17 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_pairs_desc(d_temp_storage, temp_storage_bytes, @@ -187,19 +184,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairsDescending(d_temp_storage, @@ -216,16 +213,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); ::rocprim::double_buffer d_values_db = detail::to_double_buffer(d_values); @@ -248,17 +244,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairsDescending(d_temp_storage, @@ -273,16 +269,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_keys(d_temp_storage, temp_storage_bytes, @@ -299,17 +294,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeys(d_temp_storage, @@ -324,15 +319,14 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); hipError_t error = ::rocprim::segmented_radix_sort_keys(d_temp_storage, @@ -351,16 +345,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeys(d_temp_storage, @@ -374,16 +368,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_keys_desc(d_temp_storage, temp_storage_bytes, @@ -400,17 +393,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeysDescending(d_temp_storage, @@ -425,15 +418,14 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); hipError_t error @@ -453,16 +445,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeysDescending(d_temp_storage, @@ -476,18 +468,17 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairs(d_temp_storage, temp_storage_bytes, @@ -503,19 +494,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairs(d_temp_storage, @@ -532,16 +523,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairs(d_temp_storage, temp_storage_bytes, @@ -555,17 +545,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairs(d_temp_storage, @@ -580,18 +570,18 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairsDescending(d_temp_storage, temp_storage_bytes, @@ -607,19 +597,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairsDescending(d_temp_storage, @@ -636,16 +626,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairsDescending(d_temp_storage, temp_storage_bytes, @@ -659,17 +649,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairsDescending(d_temp_storage, @@ -684,16 +674,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeys(d_temp_storage, temp_storage_bytes, @@ -707,17 +696,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeys(d_temp_storage, @@ -732,15 +721,14 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeys(d_temp_storage, temp_storage_bytes, @@ -753,16 +741,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeys(d_temp_storage, @@ -776,16 +764,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeysDescending(d_temp_storage, temp_storage_bytes, @@ -799,17 +787,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysDescending(d_temp_storage, @@ -824,15 +812,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeysDescending(d_temp_storage, temp_storage_bytes, @@ -845,16 +833,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int64_t num_items, - int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t + StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int num_items, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysDescending(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp index 5360ed4220f4..090b0f4a285d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,9 +33,9 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" -#include // IWYU pragma: export +#include "../thread/thread_operators.hpp" -#include _HIPCUB_STD_INCLUDE(functional) +#include // IWYU pragma: export #include @@ -352,7 +352,7 @@ class DeviceSelect d_out, d_num_selected_out, num_items, - _HIPCUB_STD::equal_to<>(), + hipcub::Equality(), stream, HIPCUB_DETAIL_DEBUG_SYNC_VALUE); } @@ -437,7 +437,7 @@ class DeviceSelect d_values_output, d_num_selected_out, num_items, - _HIPCUB_STD::equal_to<>(), + hipcub::Equality(), stream); } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_spmv.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_spmv.hpp new file mode 100644 index 000000000000..e5c75322dfbe --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_spmv.hpp @@ -0,0 +1,196 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_ROCPRIM_DEVICE_DEVICE_SPMV_HPP_ +#define HIPCUB_ROCPRIM_DEVICE_DEVICE_SPMV_HPP_ + +#include "../../../config.hpp" +#include "../../../util_deprecated.hpp" + +#include "../iterator/tex_obj_input_iterator.hpp" + +#include "../util_sync.hpp" + +#include + +BEGIN_HIPCUB_NAMESPACE + +class HIPCUB_DEPRECATED_BECAUSE("Use the hipSPARSE library instead") DeviceSpmv +{ + +public: + template ///< Signed integer type for sequence offsets + struct HIPCUB_DEPRECATED_BECAUSE("Use the rocSPARSE library instead") SpmvParams + { + ValueT* + d_values; ///< Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix A. + OffsetT* + d_row_end_offsets; ///< Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values + OffsetT* + d_column_indices; ///< Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix A. (Indices are zero-valued.) + ValueT* + d_vector_x; ///< Pointer to the array of \p num_cols values corresponding to the dense input vector x + ValueT* + d_vector_y; ///< Pointer to the array of \p num_rows values corresponding to the dense output vector y + int num_rows; ///< Number of rows of matrix A. + int num_cols; ///< Number of columns of matrix A. + int num_nonzeros; ///< Number of nonzero elements of matrix A. + ValueT alpha; ///< Alpha multiplicand + ValueT beta; ///< Beta addend-multiplicand + + ::hipcub::TexObjInputIterator t_vector_x; + }; + +static constexpr uint32_t CsrMVKernel_MaxThreads = 256; + +template +static __global__ void +CsrMVKernel(SpmvParams spmv_params) +{ + __shared__ ValueT partial; + + const int32_t row_id = blockIdx.x; + + if(threadIdx.x == 0) + { + partial = spmv_params.beta * spmv_params.d_vector_y[row_id]; + } + __syncthreads(); + + int32_t row_offset = (row_id == 0) ? (0) : (spmv_params.d_row_end_offsets[row_id - 1]); + for(uint32_t thread_offset = 0; thread_offset < spmv_params.num_cols / blockDim.x; + thread_offset++) + { + int32_t offset = row_offset + thread_offset * blockDim.x + threadIdx.x; + + if(offset < spmv_params.d_row_end_offsets[row_id]) + { + ValueT t_value = + spmv_params.alpha * + spmv_params.d_values[offset] * + spmv_params.d_vector_x[spmv_params.d_column_indices[offset]]; + + atomicAdd(&partial, t_value); + + __syncthreads(); + + if(threadIdx.x == 0) + { + spmv_params.d_vector_y[row_id] = partial; + } + } + } +} + +template +HIPCUB_DEPRECATED_BECAUSE("Use the rocSPARSE library instead") +HIPCUB_RUNTIME_FUNCTION static hipError_t CsrMV(void* d_temp_storage, + size_t& temp_storage_bytes, + ValueT* d_values, + int* d_row_offsets, + int* d_column_indices, + ValueT* d_vector_x, + ValueT* d_vector_y, + int num_rows, + int num_cols, + int num_nonzeros, + hipStream_t stream = 0) +{ + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + SpmvParams spmv_params; + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + spmv_params.d_values = d_values; + spmv_params.d_row_end_offsets = d_row_offsets + 1; + spmv_params.d_column_indices = d_column_indices; + spmv_params.d_vector_x = d_vector_x; + spmv_params.d_vector_y = d_vector_y; + spmv_params.num_rows = num_rows; + spmv_params.num_cols = num_cols; + spmv_params.num_nonzeros = num_nonzeros; + spmv_params.alpha = 1.0; + spmv_params.beta = 0.0; + + if(d_temp_storage == nullptr) + { + // Make sure user won't try to allocate 0 bytes memory, because + // hipMalloc will return nullptr when size is zero. + temp_storage_bytes = 4; + return hipError_t(0); + } else + { + size_t block_size = min(num_cols, static_cast(DeviceSpmv::CsrMVKernel_MaxThreads)); + size_t grid_size = num_rows; + + std::chrono::high_resolution_clock::time_point start; + if HIPCUB_IF_CONSTEXPR(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) + { + start = std::chrono::high_resolution_clock::now(); + } + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + CsrMVKernel<<>>(spmv_params); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + HIPCUB_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("CsrMV", block_size * grid_size, start); + } + return hipSuccess; +} + +template +HIPCUB_DEPRECATED_BECAUSE("Use the rocSPARSE library instead") +HIPCUB_RUNTIME_FUNCTION static hipError_t CsrMV(void* d_temp_storage, + size_t& temp_storage_bytes, + ValueT* d_values, + int* d_row_offsets, + int* d_column_indices, + ValueT* d_vector_x, + ValueT* d_vector_y, + int num_rows, + int num_cols, + int num_nonzeros, + hipStream_t stream, + bool /*debug_synchronous*/) +{ + return CsrMV(d_temp_storage, + temp_storage_bytes, + d_values, + d_row_offsets, + d_column_indices, + d_vector_x, + d_vector_y, + num_rows, + num_cols, + num_nonzeros, + stream); +} +}; + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_CUB_DEVICE_DEVICE_SELECT_HPP_ + diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp index 88a80c2dad6c..8311ec12c0b0 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -123,12 +123,12 @@ struct GridEvenShare HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH this->total_tiles = static_cast(hipcub::DivideAndRoundUp(num_items_, tile_items)); HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - this->grid_size = _HIPCUB_STD::min(total_tiles, max_grid_size); + this->grid_size = min(total_tiles, max_grid_size); int avg_tiles_per_block = total_tiles / grid_size; // leftover grains go to big blocks: this->big_shares = total_tiles - (avg_tiles_per_block * grid_size); - this->normal_share_items = static_cast(avg_tiles_per_block) * tile_items; - this->normal_base_offset = static_cast(big_shares) * tile_items; + this->normal_share_items = avg_tiles_per_block * tile_items; + this->normal_base_offset = big_shares * tile_items; this->big_share_items = normal_share_items + tile_items; } @@ -154,7 +154,7 @@ struct GridEvenShare { // This thread block gets a normal share of grains (avg_tiles_per_block) block_offset = normal_base_offset + (block_id * normal_share_items); - block_end = _HIPCUB_STD::min(num_items, block_offset + normal_share_items); + block_end = min(num_items, block_offset + normal_share_items); } // Else default past-the-end } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp index 3124a142c2ff..ad8e1131179d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -66,6 +66,7 @@ #include "device/device_segmented_reduce.hpp" #include "device/device_segmented_sort.hpp" #include "device/device_select.hpp" +#include "device/device_spmv.hpp" #include "device/device_transform.hpp" // Grid @@ -78,10 +79,15 @@ #include "iterator/arg_index_input_iterator.hpp" #include "iterator/cache_modified_input_iterator.hpp" #include "iterator/cache_modified_output_iterator.hpp" +#include "iterator/constant_input_iterator.hpp" +#include "iterator/counting_input_iterator.hpp" +#include "iterator/discard_output_iterator.hpp" #include "iterator/tex_obj_input_iterator.hpp" +#include "iterator/transform_input_iterator.hpp" // Thread #include "thread/thread_load.hpp" +#include "thread/thread_operators.hpp" #include "thread/thread_reduce.hpp" #include "thread/thread_scan.hpp" #include "thread/thread_search.hpp" diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp index 51754688bf2a..1acdb5072780 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,8 +32,6 @@ #include "../../../config.hpp" -#include "../util_type.hpp" - #include "iterator_category.hpp" #include "iterator_wrapper.hpp" @@ -47,7 +45,7 @@ BEGIN_HIPCUB_NAMESPACE template> + class InputValueType = typename std::iterator_traits::value_type> class ArgIndexInputIterator : public detail::IteratorWrapper< rocprim::arg_index_iterator, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/constant_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/constant_input_iterator.hpp new file mode 100644 index 000000000000..e135817dbb0a --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/constant_input_iterator.hpp @@ -0,0 +1,76 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_ROCPRIM_ITERATOR_CONSTANT_INPUT_ITERATOR_HPP_ +#define HIPCUB_ROCPRIM_ITERATOR_CONSTANT_INPUT_ITERATOR_HPP_ + +#include "../../../config.hpp" +#include "../../../util_deprecated.hpp" + +#include "iterator_category.hpp" +#include "iterator_wrapper.hpp" + +#include // IWYU pragma: export + +#include + +BEGIN_HIPCUB_NAMESPACE + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +class HIPCUB_DEPRECATED_BECAUSE( + "Use rocprim::constant_iterator or rocthrust::constant_iterator instead") ConstantInputIterator + : public detail::IteratorWrapper, + ConstantInputIterator> +{ + using Iterator = rocprim::constant_iterator; + using Base = detail::IteratorWrapper>; + +public: + using iterator_category = typename detail::IteratorCategory::type; + using self_type = typename Iterator::self_type; + + __host__ __device__ __forceinline__ ConstantInputIterator( + const typename Iterator::value_type value, const size_t index = 0) + : Base(Iterator(value, index)) + {} + + // Cast from wrapped iterator to class itself + __host__ __device__ __forceinline__ explicit ConstantInputIterator(Iterator iterator) + : Base(iterator) + {} +}; + +#endif + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_ROCPRIM_ITERATOR_CONSTANT_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/counting_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/counting_input_iterator.hpp new file mode 100644 index 000000000000..06802ffc8bc3 --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/counting_input_iterator.hpp @@ -0,0 +1,77 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_ROCPRIM_ITERATOR_COUNTING_INPUT_ITERATOR_HPP_ +#define HIPCUB_ROCPRIM_ITERATOR_COUNTING_INPUT_ITERATOR_HPP_ + +#include "../../../config.hpp" +#include "../../../util_deprecated.hpp" + +#include "iterator_category.hpp" +#include "iterator_wrapper.hpp" + +#include // IWYU pragma: export + +#include + +BEGIN_HIPCUB_NAMESPACE + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +class HIPCUB_DEPRECATED_BECAUSE( + "Use rocprim::counting_iterator or rocthrust::counting_iterator instead") CountingInputIterator + : public detail::IteratorWrapper, + CountingInputIterator> +{ + using Iterator = rocprim::counting_iterator; + using Base + = detail::IteratorWrapper>; + +public: + using iterator_category = typename detail::IteratorCategory::type; + using self_type = typename Iterator::self_type; + + __host__ __device__ __forceinline__ CountingInputIterator( + const typename Iterator::value_type value) + : Base(Iterator(value)) + {} + + // Cast from wrapped iterator to class itself + __host__ __device__ __forceinline__ explicit CountingInputIterator(Iterator iterator) + : Base(iterator) + {} +}; + +#endif + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_ROCPRIM_ITERATOR_COUNTING_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/discard_output_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/discard_output_iterator.hpp new file mode 100644 index 000000000000..4896530fb83a --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/discard_output_iterator.hpp @@ -0,0 +1,227 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_ROCPRIM_ITERATOR_DISCARD_OUTPUT_ITERATOR_HPP_ +#define HIPCUB_ROCPRIM_ITERATOR_DISCARD_OUTPUT_ITERATOR_HPP_ + +#include "../../../config.hpp" +#include "../../../util_deprecated.hpp" + +#include "iterator_category.hpp" + +#include // IWYU pragma: export + +#include +#include + +BEGIN_HIPCUB_NAMESPACE + +/** + * \addtogroup UtilIterator + * @{ + */ + + +/** + * \brief A discard iterator + */ +template +class HIPCUB_DEPRECATED_BECAUSE( + "Use rocprim::discard_iterator or rocthrust::discard_iterator instead") DiscardOutputIterator +{ +public: + // Required iterator traits + using self_type = DiscardOutputIterator; ///< My own type + using difference_type + = OffsetT; ///< Type to express the result of subtracting one iterator from another + using value_type = void; ///< The type of the element the iterator can point to + using pointer = void; ///< The type of a pointer to an element the iterator can point to + using reference = void; ///< The type of a reference to an element the iterator can point to + using iterator_category = + typename detail::IteratorCategory::type; ///< The iterator category + +private: + + OffsetT offset; + +public: + + /// Constructor + __host__ __device__ __forceinline__ DiscardOutputIterator( + OffsetT offset = 0) ///< Base offset + : + offset(offset) + {} + + /** + * @typedef self_type + * @brief Postfix increment + */ + __host__ __device__ __forceinline__ self_type operator++(int) + { + self_type retval = *this; + offset++; + return retval; + } + + /** + * @typedef self_type + * @brief Postfix increment + */ + __host__ __device__ __forceinline__ self_type operator++() + { + offset++; + return *this; + } + + /** + * @typedef self_type + * @brief Indirection + */ + __host__ __device__ __forceinline__ self_type& operator*() + { + // return self reference, which can be assigned to anything + return *this; + } + + /** + * @typedef self_type + * @brief Addition + */ + template + __host__ __device__ __forceinline__ self_type operator+(Distance n) const + { + self_type retval(offset + n); + return retval; + } + + /** + * @typedef self_type + * @brief Addition assignment + */ + template + __host__ __device__ __forceinline__ self_type& operator+=(Distance n) + { + offset += n; + return *this; + } + + /** + * @typedef self_type + * @brief Subtraction assignment + */ + template + __host__ __device__ __forceinline__ self_type operator-(Distance n) const + { + self_type retval(offset - n); + return retval; + } + + /** + * @typedef self_type + * @brief Subtraction assignment + */ + template + __host__ __device__ __forceinline__ self_type& operator-=(Distance n) + { + offset -= n; + return *this; + } + + /** + * @typedef self_type + * @brief Distance + */ + __host__ __device__ __forceinline__ difference_type operator-(self_type other) const + { + return offset - other.offset; + } + + /** + * @typedef self_type + * @brief Array subscript + */ + template + __host__ __device__ __forceinline__ self_type& operator[](Distance) + { + // return self reference, which can be assigned to anything + return *this; + } + + /// Structure dereference + __host__ __device__ __forceinline__ pointer operator->() + { + return; + } + + /// Assignment to anything else (no-op) + template + __host__ __device__ __forceinline__ void operator=(T const&) + {} + + /// Cast to void* operator + __host__ __device__ __forceinline__ operator void*() const + { + return nullptr; + } + + /** + * @typedef self_type + * @brief Equal to + */ + __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) const + { + return (offset == rhs.offset); + } + + /** + * @typedef self_type + * @brief Not equal to + */ + __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) const + { + return (offset != rhs.offset); + } + + /** + * @typedef self_type + * @brief ostream operator + */ + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + friend std::ostream& operator<<(std::ostream& os, const self_type& itr) + { + os << "[" << itr.offset << "]"; + return os; + } + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +}; + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_ROCPRIM_ITERATOR_DISCARD_OUTPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/transform_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/transform_input_iterator.hpp new file mode 100644 index 000000000000..2b1efba7bf2b --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/transform_input_iterator.hpp @@ -0,0 +1,87 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_ROCPRIM_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ +#define HIPCUB_ROCPRIM_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ + +#include "../../../config.hpp" + +#include "iterator_category.hpp" +#include "iterator_wrapper.hpp" + +#include // IWYU pragma: export +#include // IWYU pragma: export + +#include +#include +#include + +BEGIN_HIPCUB_NAMESPACE + +#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document + +template +class HIPCUB_DEPRECATED_BECAUSE( + "Use rocprim::transform_iterator or rocthrust::transform_iterator instead") + TransformInputIterator + : public detail::IteratorWrapper< + rocprim::transform_iterator, + TransformInputIterator> +{ + using Iterator = rocprim::transform_iterator; + using Base = detail::IteratorWrapper< + Iterator, + TransformInputIterator>; + +public: + using iterator_category = typename detail::IteratorCategory::type; + using self_type = typename Iterator::self_type; + using unary_function = typename Iterator::unary_function; + + __host__ __device__ __forceinline__ TransformInputIterator(InputIteratorT iterator, + ConversionOp transform) + : Base(Iterator(iterator, transform)) + {} + + // Cast from wrapped iterator to class itself + __host__ __device__ __forceinline__ explicit TransformInputIterator(Iterator iterator) + : Base(iterator) + {} +}; + +#endif + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_ROCPRIM_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp index e06e14edc945..655814697d4f 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -77,10 +77,10 @@ HIPCUB_FORCEINLINE T ThreadLoadVolatilePointer(T* ptr, Fundamental /* is_fundame template HIPCUB_DEVICE -HIPCUB_FORCEINLINE detail::it_value_t - ThreadLoad(InputIteratorT itr, - detail::int_constant_t /*modifier*/, - ::std::false_type /*is_pointer*/) +HIPCUB_FORCEINLINE typename std::iterator_traits::value_type + ThreadLoad(InputIteratorT itr, + detail::int_constant_t /*modifier*/, + ::std::false_type /*is_pointer*/) { return rocprim::thread_load(itr); } @@ -96,11 +96,12 @@ HIPCUB_FORCEINLINE T ThreadLoad(T* ptr, template HIPCUB_DEVICE -HIPCUB_FORCEINLINE detail::it_value_t ThreadLoad(InputIteratorT itr) +HIPCUB_FORCEINLINE + typename std::iterator_traits::value_type ThreadLoad(InputIteratorT itr) { return ThreadLoad(itr, detail::int_constant_t(), - ::std::bool_constant<_HIPCUB_STD::is_pointer::value>()); + ::std::bool_constant<::std::is_pointer::value>()); } END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp index 68be944b0315..193817f486e3 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,14 +38,12 @@ #include // IWYU pragma: export #include -#include _HIPCUB_STD_INCLUDE(functional) - #include BEGIN_HIPCUB_NAMESPACE -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::equal_to instead.") Equality +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +struct Equality { template HIPCUB_HOST_DEVICE inline constexpr bool operator()(T&& t, U&& u) const @@ -54,8 +52,8 @@ struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::equal_to instead.") Equality } }; -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") Inequality +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +struct Inequality { template HIPCUB_HOST_DEVICE inline constexpr bool operator()(T&& t, U&& u) const @@ -64,9 +62,9 @@ struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") Inequ } }; -//! deprecated [Since 5.0] -template -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") InequalityWrapper +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +template +struct InequalityWrapper { EqualityOp op; @@ -80,8 +78,8 @@ struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") Inequ } }; -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::plus instead.") Sum +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +struct Sum { template HIPCUB_HOST_DEVICE inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) @@ -90,8 +88,8 @@ struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::plus instead.") Sum } }; -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::minus instead") Difference +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +struct Difference { template HIPCUB_HOST_DEVICE inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) @@ -100,8 +98,8 @@ struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::minus instead") Difference } }; -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::divides instead") Division +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +struct Division { template HIPCUB_HOST_DEVICE inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) @@ -110,24 +108,26 @@ struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::divides instead") Division } }; -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::maximum instead.") Max +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +struct Max { template HIPCUB_HOST_DEVICE inline constexpr typename std::common_type::type operator()(T&& t, U&& u) const { + // TODO: change to use hip::std::max after libhipcxx is ready return (((u) > (t)) ? (u) : (t)); } }; -//! deprecated [Since 5.0] -struct HIPCUB_DEPRECATED_BECAUSE("Use hip::minimum instead.") Min +// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx +struct Min { template HIPCUB_HOST_DEVICE inline constexpr typename std::common_type::type operator()(T&& t, U&& u) const { + // TODO: change to use hip::std::min after libhipcxx is ready return (((u) < (t)) ? (u) : (t)); } }; @@ -256,6 +256,30 @@ struct ReduceByKeyOp } }; +template +struct BinaryFlip +{ + BinaryOpT binary_op; + + HIPCUB_HOST_DEVICE + explicit BinaryFlip(BinaryOpT binary_op) : binary_op(binary_op) + { + } + + template + HIPCUB_DEVICE auto operator()(T&& t, U&& u) -> decltype(auto) + { + return binary_op(std::forward(u), std::forward(t)); + } +}; + +template +HIPCUB_HOST_DEVICE +BinaryFlip MakeBinaryFlip(BinaryOpT binary_op) +{ + return BinaryFlip(binary_op); +} + namespace internal { @@ -276,7 +300,7 @@ struct [[deprecated( constexpr uint32_t operator()(int32_t t, int32_t u) const { - return _HIPCUB_STD::min(t, u); + return HIPCUB_MIN(t, u); } }; @@ -290,7 +314,7 @@ struct [[deprecated( constexpr uint32_t operator()(uint32_t t, uint32_t u) const { - return _HIPCUB_STD::min(t, u); + return HIPCUB_MIN(t, u); } }; @@ -306,7 +330,7 @@ struct [[deprecated( __half2 operator()(__half2 t, __half2 u) const { - return _HIPCUB_STD::min(t, u); + return HIPCUB_MIN(t, u); } }; #endif // !defined(__HIP_NO_HALF_OPERATORS__) @@ -322,7 +346,7 @@ struct [[deprecated("SIMD intrinsics are currently not supported on HIP, use Min __hip_bfloat162 operator()(__hip_bfloat162 t, __hip_bfloat162 u) const { - return _HIPCUB_STD::min(t, u); + return HIPCUB_MIN(t, u); } }; @@ -343,7 +367,7 @@ struct [[deprecated( constexpr uint32_t operator()(int32_t t, int32_t u) const { - return _HIPCUB_STD::max(t, u); + return HIPCUB_MAX(t, u); } }; @@ -357,7 +381,7 @@ struct [[deprecated( constexpr uint32_t operator()(uint32_t t, uint32_t u) const { - return _HIPCUB_STD::max(t, u); + return HIPCUB_MAX(t, u); } }; @@ -372,7 +396,7 @@ struct [[deprecated( __half2 operator()(__half2 t, __half2 u) const { - return _HIPCUB_STD::max(t, u); + return HIPCUB_MAX(t, u); } }; #endif // !defined(__HIP_NO_HALF_OPERATORS__) @@ -387,7 +411,7 @@ struct [[deprecated("SIMD intrinsics are currently not supported on HIP, use Max __hip_bfloat162 operator()(__hip_bfloat162 t, __hip_bfloat162 u) const { - return _HIPCUB_STD::max(t, u); + return HIPCUB_MAX(t, u); } }; @@ -559,7 +583,7 @@ namespace detail // Non-void value type. template using non_void_value_t = - typename std::conditional, FallbackT, IteratorT>::type; + typename std::conditional::value, FallbackT, IteratorT>::type; /// Intermediate accumulator type. template @@ -570,16 +594,16 @@ using accumulator_t = ::rocprim::accumulator_t; // // /// The output value type // using OutputT = -// typename If<(Equals, +// typename If<(Equals::value_type, // void>::VALUE), // OutputT = (if output iterator's value type is void) ? -// it_value_t< -// InputIteratorT>, // ... then the input iterator's value type, -// it_value_t>:: +// typename std::iterator_traits< +// InputIteratorT>::value_type, // ... then the input iterator's value type, +// typename std::iterator_traits::value_type>:: // Type; // ... else the output iterator's value type // // rocPRIM (as well as Thrust) uses result type of BinaryFunction instead (if not void): // -// using input_type = detail::it_value_t; +// using input_type = typename std::iterator_traits::value_type; // using result_type = ::rocprim::accumulator_t; // // For short -> float using Sum() @@ -594,8 +618,8 @@ template< > struct convert_result_type_wrapper { - using input_type = detail::it_value_t; - using output_type = it_value_t; + using input_type = typename std::iterator_traits::value_type; + using output_type = typename std::iterator_traits::value_type; using result_type = non_void_value_t; convert_result_type_wrapper(BinaryFunction op) : op(op) {} @@ -636,7 +660,7 @@ convert_result_type(BinaryFunction op) template struct convert_binary_result_type_wrapper { - using input_type = detail::it_value_t; + using input_type = typename std::iterator_traits::value_type; using init_type = InitT; using accum_type = accumulator_t; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp index b9188933220f..7547974fc008 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,7 +45,7 @@ AccumType { AccumType retval = static_cast(prefix); constexpr int length = ::hipcub::detail::static_size_v(); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int i = 0; i < length; ++i) { retval = reduction_op(retval, input[i]); @@ -60,7 +60,7 @@ AccumType ThreadReduceSequential(const InputType& input, ReductionOp reduction_o { AccumType retval = input[0]; constexpr int length = ::hipcub::detail::static_size_v(); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int i = 1; i < length; ++i) { retval = reduction_op(retval, input[i]); @@ -76,10 +76,10 @@ __device__ __forceinline__ AccumType ThreadReduceBinaryTree(const InputType& input, ReductionOp reduction_op) { constexpr auto length = ::hipcub::detail::static_size_v(); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int i = 1; i < length; i *= 2) { - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int j = 0; j + i < length; j += i * 2) { input[j] = reduction_op(input[j], input[j + i]); @@ -95,10 +95,10 @@ __device__ __forceinline__ AccumType ThreadReduceTernaryTree(const InputType& input, ReductionOp reduction_op) { constexpr auto length = ::hipcub::detail::static_size_v(); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int i = 1; i < length; i *= 3) { - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int j = 0; j + i < length; j += i * 3) { auto value = reduction_op(input[j], input[j + i]); diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp index ae32a2c84063..4995d72382e6 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -61,7 +61,7 @@ HIPCUB_FORCEINLINE ScanOp scan_op, ///< [in] Binary scan operator detail::int_constant_t /*length*/) { - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int i = 0; i < LENGTH; ++i) { inclusive = scan_op(exclusive, input[i]); @@ -144,7 +144,7 @@ HIPCUB_FORCEINLINE ScanOp scan_op, ///< [in] Binary scan operator detail::int_constant_t /*length*/) { - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int i = 0; i < LENGTH; ++i) { inclusive = scan_op(inclusive, input[i]); diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp index 6fca005dc50f..f383d0b4386f 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,8 +32,6 @@ #include "../../../config.hpp" -#include _HIPCUB_STD_INCLUDE(functional) - #include BEGIN_HIPCUB_NAMESPACE @@ -57,8 +55,8 @@ __host__ __device__ __forceinline__ void MergePathSearch( OffsetT b_len, CoordinateT& path_coordinate) { - OffsetT split_min = _HIPCUB_STD::max(diagonal - b_len, 0); - OffsetT split_max = _HIPCUB_STD::min(diagonal, a_len); + OffsetT split_min = CUB_MAX(diagonal - b_len, 0); + OffsetT split_max = CUB_MIN(diagonal, a_len); while (split_min < split_max) { @@ -75,7 +73,7 @@ __host__ __device__ __forceinline__ void MergePathSearch( } } - path_coordinate.x = _HIPCUB_STD::min(split_min, a_len); + path_coordinate.x = CUB_MIN(split_min, a_len); path_coordinate.y = diagonal - split_min; } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp index bf3c6937b13a..8e7a97b66f8f 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,19 +31,23 @@ #include "../../../config.hpp" -#include "../util_macro.hpp" #include "../util_ptx.hpp" #include "../util_type.hpp" #include // IWYU pragma: export -#include +BEGIN_HIPCUB_NAMESPACE -#if defined(__HIP_PLATFORM_NVIDIA__) - #include -#endif +// Should be deprecated once hip::std::swap is available in this scope. +template +HIPCUB_DEVICE +HIPCUB_FORCEINLINE void Swap(T& lhs, T& rhs) +{ + T temp = lhs; + lhs = rhs; + rhs = temp; +} -BEGIN_HIPCUB_NAMESPACE /** * @brief Sorts data using odd-even sort method @@ -85,28 +89,22 @@ StableOddEvenSort(KeyT (&keys)[ITEMS_PER_THREAD], { constexpr bool KEYS_ONLY = ::rocprim::Equals::VALUE; - _CCCL_SORT_MAYBE_UNROLL() - for(int i = 0; i < ITEMS_PER_THREAD; ++i) + #pragma unroll + for (int i = 0; i < ITEMS_PER_THREAD; ++i) { - _CCCL_SORT_MAYBE_UNROLL() - for(int j = 1 & i; j < ITEMS_PER_THREAD - 1; j += 2) + #pragma unroll + for (int j = 1 & i; j < ITEMS_PER_THREAD - 1; j += 2) + { + if (compare_op(keys[j + 1], keys[j])) { - if(compare_op(keys[j + 1], keys[j])) - { - -#if defined(__HIP_PLATFORM_NVIDIA__) - using ::cuda::std::swap; -#else - using ::rocprim::swap; -#endif - swap(keys[j], keys[j + 1]); - if(!KEYS_ONLY) - { - swap(items[j], items[j + 1]); - } - } - } // inner loop - } // outer loop + Swap(keys[j], keys[j + 1]); + if (!KEYS_ONLY) + { + Swap(items[j], items[j + 1]); + } + } + } // inner loop + } // outer loop } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp index 32651a8408b6..775848a122d5 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -84,7 +84,7 @@ HIPCUB_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val) ThreadStore(itr, val, detail::int_constant_t{}, - ::std::bool_constant<_HIPCUB_STD::is_pointer::value>()); + ::std::bool_constant<::std::is_pointer::value>()); } namespace detail @@ -128,5 +128,8 @@ struct iterate_thread_store } // namespace detail +template +using IterateThreadStore HIPCUB_DEPRECATED = detail::iterate_thread_store; + END_HIPCUB_NAMESPACE #endif diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp index 054811a078f1..0278f5a5cea0 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -36,12 +36,68 @@ BEGIN_HIPCUB_NAMESPACE -// RAPIDS cuDF needs to avoid unrolling some loops in sort to prevent compile time issues -#if defined(CCCL_AVOID_SORT_UNROLL) - #define _CCCL_SORT_MAYBE_UNROLL() _CCCL_PRAGMA_NOUNROLL() -#else // ^^^ CCCL_AVOID_SORT_UNROLL ^^^ / vvv !CCCL_AVOID_SORT_UNROLL vvv - #define _CCCL_SORT_MAYBE_UNROLL() _CCCL_PRAGMA_UNROLL_FULL() -#endif // !CCCL_AVOID_SORT_UNROLL +/** + * \addtogroup UtilModule + * @{ + */ + +#ifndef DOXYGEN_SHOULD_SKIP_THIS + #define HIPCUB_PREVENT_MACRO_SUBSTITUTION +template +constexpr __host__ __device__ +auto min HIPCUB_PREVENT_MACRO_SUBSTITUTION(T&& t, U&& u) + -> decltype(t < u ? std::forward(t) : std::forward(u)) +{ + return t < u ? std::forward(t) : std::forward(u); +} + +template +constexpr __host__ __device__ +auto max HIPCUB_PREVENT_MACRO_SUBSTITUTION(T&& t, U&& u) + -> decltype(t < u ? std::forward(u) : std::forward(t)) +{ + return t < u ? std::forward(u) : std::forward(t); +} + #undef HIPCUB_PREVENT_MACRO_SUBSTITUTION +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_MAX + /// Select maximum(a, b) + #define HIPCUB_MAX(a, b) (((b) > (a)) ? (b) : (a)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_MIN + /// Select minimum(a, b) + #define HIPCUB_MIN(a, b) (((b) < (a)) ? (b) : (a)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_QUOTIENT_FLOOR + /// Quotient of x/y rounded down to nearest integer + #define HIPCUB_QUOTIENT_FLOOR(x, y) ((x) / (y)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_QUOTIENT_CEILING + /// Quotient of x/y rounded up to nearest integer + #define HIPCUB_QUOTIENT_CEILING(x, y) (((x) + (y)-1) / (y)) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_ROUND_UP_NEAREST + /// x rounded up to the nearest multiple of y + #define HIPCUB_ROUND_UP_NEAREST(x, y) (HIPCUB_QUOTIENT_CEILING(x, y) * y) +#endif + +/// Deprecated since rocm [7.1] +#ifndef HIPCUB_ROUND_DOWN_NEAREST + /// x rounded down to the nearest multiple of y + #define HIPCUB_ROUND_DOWN_NEAREST(x, y) (((x) / (y)) * y) +#endif + +/** @} */ // end group UtilModule END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp index b4484569c533..3df30881bcd6 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -44,6 +44,12 @@ BEGIN_HIPCUB_NAMESPACE // * ThreadExit - not supported // * LogicShiftLeft // * LogicShiftRight +// * ThreadTrap - not supported, deprecated in CUB +// * FFMA_RZ, FMUL_RZ - not supported, deprecated in CUB +// * SHFL_IDX_SYNC - not supported, deprecated in CUB +// * WARP_SYNC - deprecated, deprecated in CUB +// * CTA_SYNC_AND - not supported, deprecated in CUB +// * CTA_SYNC_OR - not supported, deprecated in CUB // * MatchAny - not in CUB public API // // Differences: @@ -63,6 +69,18 @@ HIPCUB_FORCEINLINE int RowMajorTid(int block_dim_x, int block_dim_y, int block_d + hipThreadIdx_x; } +HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::lane_id() instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int LaneId() +{ + return ::rocprim::lane_id(); +} + +HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::warp_id instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int WarpId() +{ + return ::rocprim::warp_id(); +} + template HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t WarpMask(unsigned int warp_id) @@ -79,6 +97,34 @@ HIPCUB_FORCEINLINE uint64_t WarpMask(unsigned int warp_id) return member_mask; } +// Returns the warp lane mask of all lanes less than the calling thread +HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_lt instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskLt() +{ + return (uint64_t(1) << ::rocprim::lane_id()) - 1; +} + +// Returns the warp lane mask of all lanes less than or equal to the calling thread +HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_le instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskLe() +{ + return ((uint64_t(1) << ::rocprim::lane_id()) << 1) - 1; +} + +// Returns the warp lane mask of all lanes greater than the calling thread +HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_gt instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskGt() +{ + return uint64_t(-1)^LaneMaskLe(); +} + +// Returns the warp lane mask of all lanes greater than or equal to the calling thread +HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_ge instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskGe() +{ + return uint64_t(-1)^LaneMaskLt(); +} + // Shuffle funcs template @@ -121,6 +167,22 @@ HIPCUB_FORCEINLINE T ShuffleIndex(T input, int src_lane, unsigned int member_mas ); } +// Other + +HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") +HIPCUB_DEVICE HIPCUB_FORCEINLINE + unsigned int SHR_ADD(unsigned int x, unsigned int shift, unsigned int addend) +{ + return (x >> shift) + addend; +} + +HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") +HIPCUB_DEVICE HIPCUB_FORCEINLINE + unsigned int SHL_ADD(unsigned int x, unsigned int shift, unsigned int addend) +{ + return (x << shift) + addend; +} + namespace detail { template @@ -155,23 +217,20 @@ HIPCUB_FORCEINLINE auto // Extracts \p num_bits from \p source starting at bit-offset \p bit_start. // The input \p source may be an 8b, 16b, 32b, or 64b unsigned integer type. template -//! deprecated [Since 5.0] -HIPCUB_DEPRECATED_BECAUSE("Use hip::bitfield_extract()") HIPCUB_DEVICE -HIPCUB_FORCEINLINE - unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits) +HIPCUB_FORCEINLINE unsigned int + BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits) { static_assert(std::is_unsigned::value, "UnsignedBits must be unsigned"); return detail::unsigned_bit_extract(source, bit_start, num_bits); } -#if _CCCL_HAS_INT128() +#if HIPCUB_IS_INT128_ENABLED /** * Bitfield-extract for 128-bit types. */ template -//! deprecated [Since 5.0] -HIPCUB_DEPRECATED_BECAUSE("Use hip::bitfield_extract()") HIPCUB_DEVICE +HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, @@ -182,6 +241,78 @@ HIPCUB_FORCEINLINE unsigned int BFE(UnsignedBits source, } #endif +// Bitfield insert. +// Inserts the \p num_bits least significant bits of \p y into \p x at bit-offset \p bit_start. +HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") +HIPCUB_DEVICE HIPCUB_FORCEINLINE void BFI(unsigned int& ret, + unsigned int x, + unsigned int y, + unsigned int bit_start, + unsigned int num_bits) +{ + #ifdef __HIP_PLATFORM_AMD__ + ret = __bitinsert_u32(x, y, bit_start, num_bits); + #else + x <<= bit_start; + unsigned int MASK_X = ((1 << num_bits) - 1) << bit_start; + unsigned int MASK_Y = ~MASK_X; + ret = (y & MASK_Y) | (x & MASK_X); + #endif // __HIP_PLATFORM_AMD__ +} + +HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") +HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int IADD3(unsigned int x, unsigned int y, unsigned int z) +{ + return x + y + z; +} + +HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") +HIPCUB_DEVICE HIPCUB_FORCEINLINE int PRMT(unsigned int a, unsigned int b, unsigned int index) +{ + return ::__byte_perm(a, b, index); +} + +HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") +HIPCUB_DEVICE HIPCUB_FORCEINLINE void BAR(int count) +{ + (void) count; + __syncthreads(); +} + +HIPCUB_DEPRECATED_BECAUSE("use __syncthreads() instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE void CTA_SYNC() +{ + __syncthreads(); +} + +HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::wave_barrier() instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE void WARP_SYNC(unsigned int member_mask) +{ + (void) member_mask; + ::rocprim::wave_barrier(); +} + +HIPCUB_DEPRECATED_BECAUSE("use ::__any(predicate) instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE int WARP_ANY(int predicate, uint64_t member_mask) +{ + (void) member_mask; + return ::__any(predicate); +} + +HIPCUB_DEPRECATED_BECAUSE("use ::__all(predicate) instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE int WARP_ALL(int predicate, uint64_t member_mask) +{ + (void) member_mask; + return ::__all(predicate); +} + +HIPCUB_DEPRECATED_BECAUSE("use ::__ballot(predicate) instead") +HIPCUB_DEVICE HIPCUB_FORCEINLINE int64_t WARP_BALLOT(int predicate, uint64_t member_mask) +{ + (void) member_mask; + return __ballot(predicate); +} + namespace detail { diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp index 8a798b3ead67..0ea8de316417 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - * Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,7 +41,7 @@ { \ return _error; \ } \ - if constexpr(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) \ + if HIPCUB_IF_CONSTEXPR(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) \ { \ std::cout << name << "(" << size << ")"; \ auto __error = hipStreamSynchronize(stream); \ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp index f5bef3e3c4b5..c2e465393daf 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,7 +42,7 @@ BEGIN_HIPCUB_NAMESPACE namespace detail { // Base case: When N == 0 -template +template HIPCUB_HOST_DEVICE typename std::enable_if::type generate_partition(void* d_temp_storage, size_t& temp_storage_bytes, @@ -56,7 +56,7 @@ typename std::enable_if::type generate_partition(void* d_t } // Recursive case: When N > 0 -template +template HIPCUB_HOST_DEVICE typename std::enable_if<(N > 0), hipError_t>::type generate_partition(void* d_temp_storage, size_t& temp_storage_bytes, Generator gen, Ts... args) diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp index c17208595c9e..52ae8a3b3dfc 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,9 +42,7 @@ #include #include -#include _HIPCUB_STD_INCLUDE(limits) -#include _HIPCUB_STD_INCLUDE(iterator) - +#include #include BEGIN_HIPCUB_NAMESPACE @@ -55,12 +53,9 @@ using NullType = ::rocprim::empty_type; #endif -// This API needs to be deprecated once libhipcxx is available. -#if defined(__SIZEOF_INT128__) - #define _CCCL_HAS_INT128() 1 -#else - #define _CCCL_HAS_INT128() 0 -#endif +#ifndef HIPCUB_IS_INT128_ENABLED + #define HIPCUB_IS_INT128_ENABLED 1 +#endif // !defined(HIPCUB_IS_INT128_ENABLED) template struct [[deprecated("[Since 1.16] If is deprecated use std::conditional instead.")]] If @@ -95,30 +90,6 @@ struct PowerOfTwo namespace detail { -// the following iterator helpers are not named iter_value_t etc, like the C++20 facilities, because they are defined in -// terms of C++17 iterator_traits and not the new C++20 indirectly_readable trait etc. This allows them to detect nested -// value_type, difference_type and reference aliases, which the new C+20 traits do not consider (they only consider -// specializations of iterator_traits). Also, a value_type of void remains supported (needed by some output iterators). - -template -using it_value_t = typename _HIPCUB_STD::iterator_traits::value_type; - -template -using it_reference_t = typename _HIPCUB_STD::iterator_traits::reference; - -template -using it_difference_t = typename _HIPCUB_STD::iterator_traits::difference_type; - -template -using it_pointer_t = typename _HIPCUB_STD::iterator_traits::pointer; - -// use this whenever you need to lazily evaluate a trait. E.g., as an alternative in replace_if_use_default. -template typename Trait, typename... Args> -struct lazy_trait -{ - using type = Trait; -}; - template struct Log2Impl { @@ -176,6 +147,12 @@ struct DoubleBuffer } }; +template +struct HIPCUB_DEPRECATED_BECAUSE("Use ::std::integral_constant instead") Int2Type +{ + enum {VALUE = A}; +}; + #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document template< @@ -219,9 +196,8 @@ using is_integral_or_enum = } -// CUB removed this API, and suggests to use `::cuda::ceil_div` instead, +// CUB deprecated this API, and suggests to use `::cuda::ceil_div` instead, // which is implemented in file `libcudacxx/include/cuda/__cmath/ceil_div.h`. -// Remove when hip::ceil_div is implemented. template HIPCUB_DEPRECATED_BECAUSE("Use hip::ceil_div instead from 'libhipcxx'") HIPCUB_HOST_DEVICE __forceinline__ constexpr NumeratorT @@ -491,9 +467,9 @@ struct Uninitialized /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T using DeviceWord = typename UnitWord::DeviceWord; - static constexpr size_t DATA_SIZE = sizeof(T); - static constexpr size_t WORD_SIZE = sizeof(DeviceWord); - static constexpr size_t WORDS = DATA_SIZE / WORD_SIZE; + static constexpr std::size_t DATA_SIZE = sizeof(T); + static constexpr std::size_t WORD_SIZE = sizeof(DeviceWord); + static constexpr std::size_t WORDS = DATA_SIZE / WORD_SIZE; /// Backing storage DeviceWord storage[WORDS]; @@ -516,7 +492,7 @@ struct Uninitialized * This enum is deprecated, please use instead. Or if you have * libhipcxx, please use the type_traits system in libhipcxx. */ -enum Category +enum HIPCUB_DEPRECATED_BECAUSE("Use instead.") Category { NOT_A_NUMBER, SIGNED_INTEGER, @@ -524,27 +500,48 @@ enum Category FLOATING_POINT }; -namespace detail -{ -struct is_primitive_impl; - -template +/** + * \brief Basic type traits + */ +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +template struct BaseTraits { -private: - friend struct is_primitive_impl; - - static constexpr bool is_primitive = _PRIMITIVE; + /// Category + HIPCUB_DEPRECATED_BECAUSE("Use instead.") + static constexpr Category CATEGORY = _CATEGORY; + enum + { + PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = _PRIMITIVE, + nullptr_TYPE = _nullptr_TYPE, + }; }; +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -template -struct BaseTraits +/** + * Basic type traits (unsigned primitive specialization) + */ +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +template +struct BaseTraits { + using UnsignedBits = _UnsignedBits; - using UnsignedBits = _UnsignedBits; + HIPCUB_DEPRECATED_BECAUSE("Use instead.") + static constexpr Category CATEGORY = UNSIGNED_INTEGER; static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(0); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1); + enum + { + PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = true, + nullptr_TYPE = false, + }; + using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static HIPCUB_HOST_DEVICE __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -557,8 +554,6 @@ struct BaseTraits return key; } - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static HIPCUB_HOST_DEVICE __forceinline__ T Max() { UnsignedBits retval_bits = MAX_KEY; @@ -567,8 +562,6 @@ struct BaseTraits return retval; } - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static HIPCUB_HOST_DEVICE __forceinline__ T Lowest() { UnsignedBits retval_bits = LOWEST_KEY; @@ -576,22 +569,30 @@ struct BaseTraits memcpy(&retval, &retval_bits, sizeof(T)); return retval; } - -private: - friend struct is_primitive_impl; - - static constexpr bool is_primitive = true; }; +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -template -struct BaseTraits +/** + * Basic type traits (signed primitive specialization) + */ +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +template +struct BaseTraits { using UnsignedBits = _UnsignedBits; + HIPCUB_DEPRECATED_BECAUSE("Use instead.") + static constexpr Category CATEGORY = SIGNED_INTEGER; static constexpr UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); static constexpr UnsignedBits LOWEST_KEY = HIGH_BIT; static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; + enum + { + PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = true, + nullptr_TYPE = false, + }; + using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static HIPCUB_HOST_DEVICE __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -604,39 +605,103 @@ struct BaseTraits return key ^ HIGH_BIT; }; - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static HIPCUB_HOST_DEVICE __forceinline__ T Max() { UnsignedBits retval = MAX_KEY; return reinterpret_cast(retval); } - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static HIPCUB_HOST_DEVICE __forceinline__ T Lowest() { UnsignedBits retval = LOWEST_KEY; return reinterpret_cast(retval); } +}; +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + +// This API needs to be deprecated once libhipcxx is available. +template +struct FpLimits; + +// This API needs to be deprecated once libhipcxx is available. +template <> +struct FpLimits +{ + static HIPCUB_HOST_DEVICE __forceinline__ float Max() { + return std::numeric_limits::max(); + } + + static HIPCUB_HOST_DEVICE __forceinline__ float Lowest() { + return std::numeric_limits::max() * float(-1); + } +}; -private: - friend struct is_primitive_impl; +// This API needs to be deprecated once libhipcxx is available. +template <> +struct FpLimits +{ + static HIPCUB_HOST_DEVICE __forceinline__ double Max() { + return std::numeric_limits::max(); + } + + static HIPCUB_HOST_DEVICE __forceinline__ double Lowest() { + return std::numeric_limits::max() * double(-1); + } +}; + +// This API needs to be deprecated once libhipcxx is available. +template <> +struct FpLimits<__half> +{ + static HIPCUB_HOST_DEVICE __forceinline__ __half Max() { + unsigned short max_word = 0x7BFF; + return reinterpret_cast<__half&>(max_word); + } + + static HIPCUB_HOST_DEVICE __forceinline__ __half Lowest() { + unsigned short lowest_word = 0xFBFF; + return reinterpret_cast<__half&>(lowest_word); + } +}; + +// This API needs to be deprecated once libhipcxx is available. +template <> +struct FpLimits +{ + static HIPCUB_HOST_DEVICE __forceinline__ hip_bfloat16 Max() { + unsigned short max_word = 0x7F7F; + return reinterpret_cast(max_word); + } - static constexpr bool is_primitive = true; + static HIPCUB_HOST_DEVICE __forceinline__ hip_bfloat16 Lowest() { + unsigned short lowest_word = 0xFF7F; + return reinterpret_cast(lowest_word); + } }; -template -struct BaseTraits +/** + * Basic type traits (fp primitive specialization) + */ +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +template +struct BaseTraits { using UnsignedBits = _UnsignedBits; + HIPCUB_DEPRECATED_BECAUSE("Use instead.") + static constexpr Category CATEGORY = FLOATING_POINT; static constexpr UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(-1); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); + enum + { + PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = true, + nullptr_TYPE = false, + }; + static HIPCUB_HOST_DEVICE __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) { UnsignedBits mask = (key & HIGH_BIT) ? UnsignedBits(-1) : HIGH_BIT; @@ -649,103 +714,52 @@ struct BaseTraits return key ^ mask; }; - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") - static HIPCUB_HOST_DEVICE __forceinline__ - T Max() - { - return _HIPCUB_STD::numeric_limits::max(); + static HIPCUB_HOST_DEVICE __forceinline__ T Max() { + return FpLimits::Max(); } - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") - static HIPCUB_HOST_DEVICE __forceinline__ - T Lowest() - { - return _HIPCUB_STD::numeric_limits::lowest(); + static HIPCUB_HOST_DEVICE __forceinline__ T Lowest() { + return FpLimits::Lowest(); } - -private: - friend struct is_primitive_impl; - - static constexpr bool is_primitive = true; }; -} // namespace detail - -//! Use this class as base when specializing \ref NumericTraits for primitive signed/unsigned integers or floating-point -//! types. -template -using BaseTraits = detail::BaseTraits<_CATEGORY, _PRIMITIVE, _UnsignedBits, T>; - -//! Numeric type traits for radix sort key operations, decoupled lookback and tuning. You can specialize this template -//! for your own types if: -//! * There is an unsigned integral type of equal size -//! * The size of the type is smaller than 64bits -//! * The arithmetic throughput of the type is similar to other built-in types of the same size -//! For other types, if you want to use them with radix sort, please use the decomposer interface of the radix sort. +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -template -struct NumericTraits : BaseTraits -{}; +/** + * \brief Numeric type traits + */ +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +template struct NumericTraits : BaseTraits {}; -template<> -struct NumericTraits : BaseTraits -{}; +template <> struct NumericTraits : BaseTraits {}; -template<> -struct NumericTraits - : BaseTraits<(_HIPCUB_STD::numeric_limits::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, - true, - unsigned char, - char> -{}; -template<> -struct NumericTraits : BaseTraits -{}; -template<> -struct NumericTraits : BaseTraits -{}; -template<> -struct NumericTraits : BaseTraits -{}; -template<> -struct NumericTraits : BaseTraits -{}; -template<> -struct NumericTraits : BaseTraits -{}; +template <> struct NumericTraits : BaseTraits<(std::numeric_limits::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, true, false, unsigned char, char> {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; -template<> -struct NumericTraits - : BaseTraits -{}; -template<> -struct NumericTraits - : BaseTraits -{}; -template<> -struct NumericTraits - : BaseTraits -{}; -template<> -struct NumericTraits - : BaseTraits -{}; -template<> -struct NumericTraits - : BaseTraits -{}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; - #if _CCCL_HAS_INT128() + #if HIPCUB_IS_INT128_ENABLED template<> struct NumericTraits<__uint128_t> { using T = __uint128_t; using UnsignedBits = __uint128_t; + static constexpr Category CATEGORY = UNSIGNED_INTEGER; static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(0); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1); + HIPCUB_DEPRECATED_BECAUSE("Use instead.") + static constexpr bool PRIMITIVE = false; + static constexpr bool nullptr_TYPE = false; + using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static __host__ __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -758,15 +772,11 @@ struct NumericTraits<__uint128_t> return key; } - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static __host__ __device__ __forceinline__ T Max() { return MAX_KEY; } - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static __host__ __device__ __forceinline__ T Lowest() { return LOWEST_KEY; @@ -779,10 +789,15 @@ struct NumericTraits<__int128_t> using T = __int128_t; using UnsignedBits = __uint128_t; + static constexpr Category CATEGORY = SIGNED_INTEGER; static constexpr UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); static constexpr UnsignedBits LOWEST_KEY = HIGH_BIT; static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; + HIPCUB_DEPRECATED_BECAUSE("Use instead.") + static constexpr bool PRIMITIVE = false; + static constexpr bool nullptr_TYPE = false; + using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static __host__ __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -795,86 +810,42 @@ struct NumericTraits<__int128_t> return key ^ HIGH_BIT; }; - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static __host__ __device__ __forceinline__ T Max() { UnsignedBits retval = MAX_KEY; return reinterpret_cast(retval); } - //! deprecated [Since 5.0] - HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static __host__ __device__ __forceinline__ T Lowest() { UnsignedBits retval = LOWEST_KEY; return reinterpret_cast(retval); } - -private: - friend struct detail::is_primitive_impl; - - static constexpr bool is_primitive = false; }; #endif -template<> -struct NumericTraits : BaseTraits -{}; -template<> -struct NumericTraits : BaseTraits -{}; -template<> -struct NumericTraits<__half> : BaseTraits -{ - using UnsignedBits = unsigned short; -}; -template<> -struct NumericTraits - : BaseTraits -{ - using UnsignedBits = unsigned short; -}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; +template <> struct NumericTraits<__half> : BaseTraits {}; +template <> struct NumericTraits : BaseTraits {}; -template<> -struct NumericTraits - : BaseTraits::VolatileWord, bool> -{}; +template <> struct NumericTraits : BaseTraits::VolatileWord, bool> {}; +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -namespace detail -{ +/** + * \brief Type traits + */ template struct Traits : NumericTraits::type> {}; -} // namespace detail - -//! \brief Query type traits for radix sort key operations, decoupled lookback and tunings. To add support for your own -//! primitive types please specialize \ref NumericTraits. -template -using Traits = detail::Traits; - namespace detail { -// we cannot befriend is_primitive on GCC < 11, since it's a template (bug) -struct is_primitive_impl -{ - // must be a struct instead of an alias, so the access of Traits::is_primitive happens in the context of this class - template - struct is_primitive : _HIPCUB_STD::bool_constant::is_primitive> - {}; -}; -// This trait serves two purposes: -// 1. It is used for tunings to detect whether we have a build-in arithmetic type for which we can expect certain -// arithmetic throughput. E.g.: we expect all primitive types of the same size to show roughly similar performance. -// 2. Decoupled lookback uses this trait to determine whether there is a machine word twice the size of T which can be -// loaded/stored with a single instruction. -// TODO(bgruber): for 2. we should probably just check whether sizeof(T) * 2 <= sizeof(int128) (or 256-bit on SM100) -// Users must be able to hook into both scenarios with their custom types, so this trait must depend on cub::Traits +// __uint128_t and __int128_t are not primitive HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH template -struct is_primitive : is_primitive_impl::is_primitive +struct is_primitive : ::std::bool_constant::PRIMITIVE> {}; template @@ -907,8 +878,8 @@ template struct is_extended_fp : std::integral_constant< bool, - _HIPCUB_STD::is_same<__half, typename std::remove_cv::type>::value - || _HIPCUB_STD::is_same::type>::value> + std::is_same<__half, typename std::remove_cv::type>::value + || std::is_same::type>::value> {}; // Gets "raw" type: drops reference and const qualifier. diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp index 7fac1a910fbc..20a85eab8d5f 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -63,6 +63,7 @@ using InternalWarpExchangeImpl template class WarpExchange : private detail::InternalWarpExchangeImpl struct LoadInternal { - using WarpExchangeT = WarpExchange; + using WarpExchangeT = WarpExchange< + InputT, + ITEMS_PER_THREAD, + LOGICAL_WARP_THREADS, + ARCH + >; using TempStorage = typename WarpExchangeT::TempStorage; TempStorage& temp_storage; int linear_tid; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp index c17896227c4e..3d6e938b7337 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,7 @@ #include "../../../config.hpp" #include "../util_ptx.hpp" +#include "../thread/thread_operators.hpp" #include // IWYU pragma: export diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp index b5c6071d6c7c..60a20e6ecb9d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,7 @@ #include "../../../config.hpp" #include "../util_ptx.hpp" +#include "../thread/thread_operators.hpp" #include // IWYU pragma: export diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp index a3be7d521c6d..23e0c52f6cda 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -200,7 +200,12 @@ class WarpStore template <> struct StoreInternal { - using WarpExchangeT = WarpExchange; + using WarpExchangeT = WarpExchange< + T, + ITEMS_PER_THREAD, + LOGICAL_WARP_THREADS, + ARCH + >; using TempStorage = typename WarpExchangeT::TempStorage; TempStorage& temp_storage; int linear_tid; diff --git a/projects/hipcub/hipcub/include/hipcub/config.hpp b/projects/hipcub/hipcub/include/hipcub/config.hpp index e5ed5a94e07c..18ad351a093b 100644 --- a/projects/hipcub/hipcub/include/hipcub/config.hpp +++ b/projects/hipcub/hipcub/include/hipcub/config.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2019-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2019-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -35,14 +35,6 @@ // Version #include "hipcub_version.hpp" // IWYU pragma: export -// Manage std implementation -#include "libcxx.hpp" // IWYU pragma: export - -// For _CCCL_IMPLICIT_SYSTEM_HEADER -#if _HIPCUB_HAS_DEVICE_SYSTEM_STD - #include _HIPCUB_LIBCXX_INCLUDE(__cccl_config) // IWYU pragma: export -#endif - #define HIPCUB_NAMESPACE hipcub // Inline namespace (e.g. HIPCUB_300400_NS where 300400 is the hipCUB version) is used to @@ -120,8 +112,6 @@ END_HIPCUB_NAMESPACE #define HIPCUB_RUNTIME_FUNCTION CUB_RUNTIME_FUNCTION #include - #include - #include #define HIPCUB_WARP_THREADS CUB_PTX_WARP_THREADS #define HIPCUB_DEVICE_WARP_THREADS CUB_PTX_WARP_THREADS #define HIPCUB_HOST_WARP_THREADS CUB_PTX_WARP_THREADS @@ -210,6 +200,19 @@ END_HIPCUB_NAMESPACE #define HipcubLog(msg) ::hipcub::Log(msg, __FILE__, __LINE__) #endif +#if __cpp_if_constexpr + #define HIPCUB_IF_CONSTEXPR constexpr +#else + #if defined(_MSC_VER) && !defined(__clang__) + // MSVC (and not Clang pretending to be MSVC) unconditionally exposes if constexpr (even in C++14 mode), + // moreover it triggers warning C4127 (conditional expression is constant) when not using it. nvcc will + // be calling cl.exe for host-side codegen. + #define HIPCUB_IF_CONSTEXPR constexpr + #else + #define HIPCUB_IF_CONSTEXPR + #endif +#endif + #ifdef DOXYGEN_SHOULD_SKIP_THIS // Documentation only /// \def HIPCUB_DEBUG_SYNC @@ -239,13 +242,4 @@ END_HIPCUB_NAMESPACE #endif #endif // HIPCUB_ROCPRIM_API -// This API needs to be deprecated once libhipcxx is available. -#if !defined(_CCCL_PRAGMA_UNROLL_FULL) - #define _CCCL_PRAGMA_UNROLL_FULL() _Pragma("unroll") -#endif // !defined(_CCCL_PRAGMA_UNROLL_FULL) - -#if !defined(_CCCL_PRAGMA_NOUNROLL) - #define _CCCL_PRAGMA_NOUNROLL() _Pragma("nounroll") -#endif // !defined(_CCCL_PRAGMA_NOUNROLL) - #endif // HIPCUB_CONFIG_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/device/device_spmv.hpp b/projects/hipcub/hipcub/include/hipcub/device/device_spmv.hpp new file mode 100644 index 000000000000..b32fc2811d30 --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/device/device_spmv.hpp @@ -0,0 +1,39 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_DEVICE_DEVICE_SPMV_HPP_ +#define HIPCUB_DEVICE_DEVICE_SPMV_HPP_ + +#ifdef __HIP_PLATFORM_AMD__ + #include "../backend/rocprim/device/device_spmv.hpp" // IWYU pragma: export +#elif defined(__HIP_PLATFORM_NVIDIA__) + #include "../backend/cub/device/device_spmv.hpp" // IWYU pragma: export +#endif + +#endif // HIPCUB_DEVICE_DEVICE_SELECT_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/grid/grid_barrier.hpp b/projects/hipcub/hipcub/include/hipcub/grid/grid_barrier.hpp new file mode 100644 index 000000000000..8c8863be6474 --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/grid/grid_barrier.hpp @@ -0,0 +1,40 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_GRID_GRID_BARRIER_HPP_ +#define HIPCUB_GRID_GRID_BARRIER_HPP_ + +#ifdef __HIP_PLATFORM_AMD__ + #include "../backend/rocprim/grid/grid_barrier.hpp" // IWYU pragma: export +#elif defined(__HIP_PLATFORM_NVIDIA__) + #include "../backend/cub/grid/grid_barrier.hpp" // IWYU pragma: export + #include "../config.hpp" +#endif + +#endif // HIPCUB_GRID_GRID_BARRIER_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in b/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in index 5e72747ee94d..86790382e504 100644 --- a/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in +++ b/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/constant_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/constant_input_iterator.hpp new file mode 100644 index 000000000000..7fc28707274d --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/iterator/constant_input_iterator.hpp @@ -0,0 +1,40 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_CONSTANT_INPUT_ITERATOR_HPP_ +#define HIPCUB_CONSTANT_INPUT_ITERATOR_HPP_ + +#ifdef __HIP_PLATFORM_AMD__ + #include "../backend/rocprim/iterator/constant_input_iterator.hpp" // IWYU pragma: export +#elif defined(__HIP_PLATFORM_NVIDIA__) + #include "../config.hpp" + #include // IWYU pragma: export +#endif + +#endif // HIPCUB_ITERATOR_DISCARD_OUTPUT__HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/counting_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/counting_input_iterator.hpp new file mode 100644 index 000000000000..723af464aae3 --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/iterator/counting_input_iterator.hpp @@ -0,0 +1,41 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_COUNTING_INPUT_ITERATOR_HPP_ +#define HIPCUB_COUNTING_INPUT_ITERATOR_HPP_ + +#ifdef __HIP_PLATFORM_AMD__ + #include "../backend/rocprim/iterator/counting_input_iterator.hpp" // IWYU pragma: export +#elif defined(__HIP_PLATFORM_NVIDIA__) + #include "../config.hpp" + #include // IWYU pragma: export +#endif + +#endif // HIPCUB_ITERATOR_DISCARD_OUTPUT__HPP_ + diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/discard_output_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/discard_output_iterator.hpp new file mode 100644 index 000000000000..a2c8d6bedd6c --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/iterator/discard_output_iterator.hpp @@ -0,0 +1,40 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_ITERATOR_DISCARD_OUTPUT_HPP_ +#define HIPCUB_ITERATOR_DISCARD_OUTPUT_HPP_ + +#ifdef __HIP_PLATFORM_AMD__ + #include "../backend/rocprim/iterator/discard_output_iterator.hpp" // IWYU pragma: export +#elif defined(__HIP_PLATFORM_NVIDIA__) + #include "../config.hpp" + #include // IWYU pragma: export +#endif + +#endif // HIPCUB_ITERATOR_DISCARD_OUTPUT__HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/transform_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/transform_input_iterator.hpp new file mode 100644 index 000000000000..b2ce4b9f20fa --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/iterator/transform_input_iterator.hpp @@ -0,0 +1,40 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ +#define HIPCUB_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ + +#ifdef __HIP_PLATFORM_AMD__ + #include "../backend/rocprim/iterator/transform_input_iterator.hpp" // IWYU pragma: export +#elif defined(__HIP_PLATFORM_NVIDIA__) + #include "../config.hpp" + #include // IWYU pragma: export +#endif + +#endif // HIPCUB_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/libcxx.hpp b/projects/hipcub/hipcub/include/hipcub/libcxx.hpp deleted file mode 100644 index 7ab49d3969fd..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/libcxx.hpp +++ /dev/null @@ -1,103 +0,0 @@ -// MIT License -// -// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -#ifndef HIPCUB_LIBCXX_HPP_ -#define HIPCUB_LIBCXX_HPP_ - -#pragma once - -// This is a utility file that helps managing which -// 'std' implementation we're using. The provided -// macros are for internal use only and may change -// in future versions. -// -// Example usage: -// #include _HIPCUB_STD_INCLUDE(optional) -// using optional_int = _HIPCUB_STD::optional; - -// Minimum version that we depend on. -#define _HIPCUB_REQUIRED_LIBCXX_VERSION_MAJOR 3 -#define _HIPCUB_REQUIRED_LIBCXX_VERSION_MINOR 0 -#define _HIPCUB_REQUIRED_LIBCXX_VERSION_PATCH 0 - -#define _HIPCUB_REQUIRED_LIBCXX_VERSION \ - _HIPCUB_REQUIRED_LIBCXX_VERSION_MAJOR * 1000000 + _HIPCUB_REQUIRED_LIBCXX_VERSION_MINOR * 1000 \ - + _HIPCUB_REQUIRED_LIBCXX_VERSION_PATCH - -#ifdef __has_include - #define HIPCUB_HAS_INCLUDE(_X) __has_include(_X) -#else - #define HIPCUB_HAS_INCLUDE(_X) 0 -#endif - -#define _HIPCUB_STRINGIFY_IMPL(x) #x -#define _HIPCUB_STRINGIFY(x) _HIPCUB_STRINGIFY_IMPL(x) - -// clang-format off - -// If the '::cuda::std' namespace from 'libcudacxx' or 'libhipcxx' is available. -#if HIPCUB_HAS_INCLUDE() - #include - // If version matches and '_CUDA_VSTD' is available. - #if defined(_LIBCUDACXX_CUDA_API_VERSION) && (_LIBCUDACXX_CUDA_API_VERSION >= _HIPCUB_REQUIRED_LIBCXX_VERSION) && defined(_CUDA_VSTD) - #define _HIPCUB_LIBCXX_INCLUDE(LIB) _HIPCUB_STRINGIFY(cuda/LIB) - #define _HIPCUB_STD_INCLUDE(LIB) _HIPCUB_STRINGIFY(cuda/std/LIB) - #define _HIPCUB_LIBCXX ::cuda - #define _HIPCUB_STD _CUDA_VSTD - #define _HIPCUB_HAS_DEVICE_SYSTEM_STD 1 - #define _HIPCUB_STD_NAMESPACE_BEGIN _LIBCUDACXX_BEGIN_NAMESPACE_STD - #define _HIPCUB_STD_NAMESPACE_END _LIBCUDACXX_END_NAMESPACE_STD - #endif -#endif -// Otherwise, if the '::hip::std' namespace from 'libhipcxx' is available. -#if !defined(_HIPCUB_HAS_DEVICE_SYSTEM_STD) && HIPCUB_HAS_INCLUDE() - #include - // If version matches and '_CUDA_VSTD' is available. - #if defined(_LIBCUDACXX_CUDA_API_VERSION) && (_LIBCUDACXX_CUDA_API_VERSION >= _HIPCUB_REQUIRED_LIBCXX_VERSION) && defined(_CUDA_VSTD) - #define _HIPCUB_LIBCXX_INCLUDE(LIB) _HIPCUB_STRINGIFY(hip/LIB) - #define _HIPCUB_STD_INCLUDE(LIB) _HIPCUB_STRINGIFY(hip/std/LIB) - // In 'libhipcxx' the '::hip' namespace is synonymous with '::cuda'. - #define _HIPCUB_LIBCXX ::hip - // In 'libhipcxx' the macro '_CUDA_VSTD' is also defined. - #define _HIPCUB_STD _CUDA_VSTD - #define _HIPCUB_HAS_DEVICE_SYSTEM_STD 1 - #define _HIPCUB_STD_NAMESPACE_BEGIN _LIBCUDACXX_BEGIN_NAMESPACE_STD - #define _HIPCUB_STD_NAMESPACE_END _LIBCUDACXX_END_NAMESPACE_STD - #endif -#endif - -// If 'libcudacxx' or 'libhipcxx' is not found, use fallback. -#ifndef _HIPCUB_HAS_DEVICE_SYSTEM_STD - #define _HIPCUB_LIBCXX_INCLUDE(LIB) _HIPCUB_STRINGIFY(LIB) - #define _HIPCUB_STD_INCLUDE(LIB) _HIPCUB_STRINGIFY(LIB) - #define _HIPCUB_LIBCXX - #define _HIPCUB_STD ::std - #define _HIPCUB_HAS_DEVICE_SYSTEM_STD 0 - #define _HIPCUB_STD_NAMESPACE_BEGIN \ - namespace std \ - { - #define _HIPCUB_STD_NAMESPACE_END } -#endif - -// clang-format on - -#endif // HIPCUB_LIBCXX_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp b/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp index 6dbd6ed44346..c982be81ec7c 100644 --- a/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp +++ b/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,8 +38,8 @@ #include "../backend/rocprim/thread/thread_store.hpp" // IWYU pragma: export #elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../backend/cub/thread/thread_store.hpp" // IWYU pragma: export - + #include "../config.hpp" + #include // IWYU pragma: export #endif #endif diff --git a/projects/hipcub/hipcub/include/hipcub/util_type.hpp b/projects/hipcub/hipcub/include/hipcub/util_type.hpp index 6dd78870928c..9f755ce45d62 100644 --- a/projects/hipcub/hipcub/include/hipcub/util_type.hpp +++ b/projects/hipcub/hipcub/include/hipcub/util_type.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2020-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,7 +33,7 @@ #ifdef __HIP_PLATFORM_AMD__ #include "backend/rocprim/util_type.hpp" // IWYU pragma: export #elif defined(__HIP_PLATFORM_NVIDIA__) - #include "backend/cub/util_type.hpp" // IWYU pragma: export + #include // IWYU pragma: export #endif diff --git a/projects/hipcub/rtest.xml b/projects/hipcub/rtest.xml index 7156f0dcbd34..af8b5917109a 100644 --- a/projects/hipcub/rtest.xml +++ b/projects/hipcub/rtest.xml @@ -2,10 +2,10 @@ - + - + {CTEST_FILTER} {CTEST_REGEX} diff --git a/projects/hipcub/test/CMakeLists.txt b/projects/hipcub/test/CMakeLists.txt index 70c28b2b197c..11771eae643a 100644 --- a/projects/hipcub/test/CMakeLists.txt +++ b/projects/hipcub/test/CMakeLists.txt @@ -71,16 +71,7 @@ endfunction() # We'll use this small program to detect available GPUs and build the resource JSON file. set(GEN_RES_SPEC_PATH ${CMAKE_SOURCE_DIR}/test/generate_resource_spec.cpp) add_executable(generate_resource_spec ${GEN_RES_SPEC_PATH}) -set_target_properties(generate_resource_spec - PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" -) - -if(HIP_COMPILER STREQUAL "nvcc") - set_source_files_properties(${GEN_RES_SPEC_PATH} - PROPERTIES LANGUAGE CUDA) -endif() - -target_compile_options(generate_resource_spec PRIVATE -Wno-unused-command-line-argument) +set_target_properties(generate_resource_spec PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") target_link_libraries(generate_resource_spec PRIVATE hip::host) # This test may still get the offload-compress flag passed. Since it does not include any kernel # code it will give an unused-command-line-argument warning. diff --git a/projects/hipcub/test/extra/CMakeLists.txt b/projects/hipcub/test/extra/CMakeLists.txt index 801ba1a5c0bd..8ab3eb577551 100644 --- a/projects/hipcub/test/extra/CMakeLists.txt +++ b/projects/hipcub/test/extra/CMakeLists.txt @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -42,7 +42,7 @@ include(VerifyCompiler) # CUB (only for CUDA platform) if(HIP_COMPILER STREQUAL "nvcc") - set(CCCL_MINIMUM_VERSION 3.0.0) + set(CCCL_MINIMUM_VERSION 2.8.2) if(NOT DOWNLOAD_CUB) find_package(CCCL ${CCCL_MINIMUM_VERSION} CONFIG) endif() diff --git a/projects/hipcub/test/generate_resource_spec.cpp b/projects/hipcub/test/generate_resource_spec.cpp index 77569124291e..3fae3cc1618f 100644 --- a/projects/hipcub/test/generate_resource_spec.cpp +++ b/projects/hipcub/test/generate_resource_spec.cpp @@ -10,7 +10,7 @@ // ./enum_device // // Sample output: -// { +// { // "version": { // "major": 1, // "minor": 0 @@ -113,7 +113,7 @@ int main(int argc, char* argv[]) // Add one object for each gfxID. // Each gfxID-keyed object will contain an array of device IDs. - size_t key_index = 0; + unsigned int key_index = 0; for(auto& name_it : names_to_ids) { out_file << " \"" << name_it.first << "\": [" << std::endl; @@ -124,7 +124,7 @@ int main(int argc, char* argv[]) // to have a consistent output on each run so that the resource // spec file stays the same. std::sort(name_it.second.begin(), name_it.second.end()); - size_t id_index = 0; + unsigned int id_index = 0; for(const auto& id_it : name_it.second) { out_file << " {" << std::endl; diff --git a/projects/hipcub/test/hipcub/CMakeLists.txt b/projects/hipcub/test/hipcub/CMakeLists.txt index d69d1d855c5d..f62d3d85d224 100644 --- a/projects/hipcub/test/hipcub/CMakeLists.txt +++ b/projects/hipcub/test/hipcub/CMakeLists.txt @@ -1,5 +1,5 @@ # MIT License # -# Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -257,10 +257,12 @@ add_hipcub_test_parallel("hipcub.DeviceSegmentedRadixSort" test_hipcub_device_se add_hipcub_test("hipcub.DeviceSegmentedReduce" test_hipcub_device_segmented_reduce.cpp) add_hipcub_test_parallel("hipcub.DeviceSegmentedSort" test_hipcub_device_segmented_sort.cpp.in) add_hipcub_test("hipcub.DeviceSelect" test_hipcub_device_select.cpp) +add_hipcub_test("hipcub.DeviceSpmv" test_hipcub_device_spmv.cpp) add_hipcub_test("hipcub.DeviceTransform" test_hipcub_device_transform.cpp) add_hipcub_test("hipcub.DevicePartition" test_hipcub_device_partition.cpp) add_hipcub_test("hipcub.Grid" test_hipcub_grid.cpp) add_hipcub_test("hipcub.UtilPtx" test_hipcub_util_ptx.cpp) +add_hipcub_test("hipcub.UtilDevice" test_hipcub_util_device.cpp) add_hipcub_test("hipcub.Vector" test_hipcub_vector.cpp) add_hipcub_test("hipcub.WarpExchange" test_hipcub_warp_exchange.cpp) add_hipcub_test("hipcub.WarpLoad" test_hipcub_warp_load.cpp) diff --git a/projects/hipcub/test/hipcub/bfloat16.hpp b/projects/hipcub/test/hipcub/bfloat16.hpp index 8aed6fc7c228..cc63e155b534 100644 --- a/projects/hipcub/test/hipcub/bfloat16.hpp +++ b/projects/hipcub/test/hipcub/bfloat16.hpp @@ -1,6 +1,5 @@ /****************************************************************************** * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,11 +39,7 @@ #include #if defined(__HIP_PLATFORM_NVIDIA__) - #include - #include - #include -#else - #include +#include #endif #ifdef __GNUC__ @@ -53,33 +48,6 @@ #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif -struct bfloat16_t; - -#if defined(__HIP_PLATFORM_NVIDIA__) - #include - #include - -using hip_bfloat16 = bfloat16_t; -namespace cuda -{ -namespace std -{ - -template<> -struct is_floating_point : true_type -{}; - -template<> -class numeric_limits -{ -public: - static constexpr bool is_specialized = true; -}; - -} // namespace std -} // namespace cuda - -#endif // __HIP_PLATFORM_NVIDIA__ /****************************************************************************** * bfloat16_t @@ -118,20 +86,18 @@ struct bfloat16_t *this = bfloat16_t(float(a)); } - /// Constructor from size_t - __host__ __device__ __forceinline__ - bfloat16_t(size_t a) + /// Constructor from std::size_t + __host__ __device__ __forceinline__ bfloat16_t(std::size_t a) { *this = bfloat16_t(float(a)); } /// Constructor from unsigned long long int template - && (!std::is_same_v)>::type> - __host__ __device__ __forceinline__ - bfloat16_t(T a) + typename = typename std::enable_if< + std::is_same::value + && (!std::is_same::value)>::type> + __host__ __device__ __forceinline__ bfloat16_t(T a) { *this = bfloat16_t(float(a)); } @@ -297,22 +263,40 @@ inline std::ostream& operator<<(std::ostream &out, const bfloat16_t &x) #if defined(__HIP_PLATFORM_NVIDIA__) -/// Insert formatted \p __nv_bfloat16 into the output stream -inline std::ostream& operator<<(std::ostream& out, const __nv_bfloat16& x) -{ - return out << bfloat16_t(x); -} + /// Insert formatted \p __nv_bfloat16 into the output stream + inline std::ostream& operator<<(std::ostream &out, const __nv_bfloat16 &x) + { + return out << bfloat16_t(x); + } #endif + + + /****************************************************************************** * Traits overloads ******************************************************************************/ -template<> -struct hipcub::NumericTraits - : hipcub::BaseTraits -{}; +template <> +struct hipcub::FpLimits +{ + static __host__ __device__ __forceinline__ bfloat16_t Max() { return bfloat16_t::max(); } + + static __host__ __device__ __forceinline__ bfloat16_t Lowest() { return bfloat16_t::lowest(); } +}; + +#if defined(__HIP_PLATFORM_NVIDIA__) +_CCCL_SUPPRESS_DEPRECATED_PUSH +#else +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +#endif +template <> struct hipcub::NumericTraits : hipcub::BaseTraits {}; +#if defined(__HIP_PLATFORM_NVIDIA__) +_CCCL_SUPPRESS_DEPRECATED_POP +#else +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +#endif #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/projects/hipcub/test/hipcub/common_test_header.hpp b/projects/hipcub/test/hipcub/common_test_header.hpp index 13321827b8fa..737c29142cda 100755 --- a/projects/hipcub/test/hipcub/common_test_header.hpp +++ b/projects/hipcub/test/hipcub/common_test_header.hpp @@ -22,9 +22,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -47,29 +49,19 @@ #include #endif -#include - -#include _HIPCUB_LIBCXX_INCLUDE(cmath) -#include _HIPCUB_STD_INCLUDE(limits) - // test_utils.hpp should only be included by this header. // The following definition is used as guard in test_utils.hpp // Including test_utils.hpp by itself will cause a compile error. -#define TEST_UTILS_INCLUDE_GUARD +#define TEST_UTILS_INCLUDE_GAURD #include "test_utils.hpp" -#if defined(__clang__) - #if defined(__SANITIZE_ADDRESS__) \ - || (defined(__has_feature) && __has_feature(address_sanitizer)) - #define GTEST_SKIP_ASAN() \ - do \ - { \ - GTEST_SKIP() << "Skipping test under ASan"; \ - } \ - while(0) - #else - #define GTEST_SKIP_ASAN() - #endif +#if defined(__SANITIZE_ADDRESS__) || (defined(__has_feature) && __has_feature(address_sanitizer)) + #define GTEST_SKIP_ASAN() \ + do \ + { \ + GTEST_SKIP() << "Skipping test under ASan"; \ + } \ + while(0) #else #define GTEST_SKIP_ASAN() #endif diff --git a/projects/hipcub/test/hipcub/experimental/sparse_matrix.hpp b/projects/hipcub/test/hipcub/experimental/sparse_matrix.hpp new file mode 100644 index 000000000000..941c21cae52f --- /dev/null +++ b/projects/hipcub/test/hipcub/experimental/sparse_matrix.hpp @@ -0,0 +1,1240 @@ +/****************************************************************************** + * Copyright (c) 2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +/****************************************************************************** + * Matrix data structures and parsing logic + ******************************************************************************/ + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CUB_MKL + #include + #include +#endif + +using namespace std; + +/****************************************************************************** + * COO matrix type + ******************************************************************************/ + +struct GraphStats +{ + int num_rows; + int num_cols; + int num_nonzeros; + + double diag_dist_mean; // mean + double diag_dist_std_dev; // sample std dev + double pearson_r; // coefficient of variation + + double row_length_mean; // mean + double row_length_std_dev; // sample std_dev + double row_length_variation; // coefficient of variation + double row_length_skewness; // skewness + + void Display(bool show_labels = true) + { + if (show_labels) + printf("\n" + "\t num_rows: %d\n" + "\t num_cols: %d\n" + "\t num_nonzeros: %d\n" + "\t diag_dist_mean: %.2f\n" + "\t diag_dist_std_dev: %.2f\n" + "\t pearson_r: %f\n" + "\t row_length_mean: %.5f\n" + "\t row_length_std_dev: %.5f\n" + "\t row_length_variation: %.5f\n" + "\t row_length_skewness: %.5f\n", + num_rows, + num_cols, + num_nonzeros, + diag_dist_mean, + diag_dist_std_dev, + pearson_r, + row_length_mean, + row_length_std_dev, + row_length_variation, + row_length_skewness); + else + printf( + "%d, " + "%d, " + "%d, " + "%.2f, " + "%.2f, " + "%f, " + "%.5f, " + "%.5f, " + "%.5f, " + "%.5f, ", + num_rows, + num_cols, + num_nonzeros, + diag_dist_mean, + diag_dist_std_dev, + pearson_r, + row_length_mean, + row_length_std_dev, + row_length_variation, + row_length_skewness); + } +}; + + + +/****************************************************************************** + * COO matrix type + ******************************************************************************/ + + +/** + * COO matrix type. A COO matrix is just a vector of edge tuples. Tuples are sorted + * first by row, then by column. + */ +template +struct CooMatrix +{ + //--------------------------------------------------------------------- + // Type definitions and constants + //--------------------------------------------------------------------- + + // COO edge tuple + struct CooTuple + { + OffsetT row; + OffsetT col; + ValueT val; + + CooTuple() : row(OffsetT()), col(OffsetT()), val(ValueT()) {} + CooTuple(OffsetT row, OffsetT col) : row(row), col(col) {} + CooTuple(OffsetT row, OffsetT col, ValueT val) : row(row), col(col), val(val) {} + + /** + * Comparator for sorting COO sparse format num_nonzeros + */ + bool operator<(const CooTuple &other) const + { + if ((row < other.row) || ((row == other.row) && (col < other.col))) + { + return true; + } + + return false; + } + }; + + + //--------------------------------------------------------------------- + // Data members + //--------------------------------------------------------------------- + + // Fields + int num_rows; + int num_cols; + int num_nonzeros; + CooTuple* coo_tuples; + + //--------------------------------------------------------------------- + // Methods + //--------------------------------------------------------------------- + + // Constructor + CooMatrix() : num_rows(0), num_cols(0), num_nonzeros(0), coo_tuples(nullptr) {} + + /** + * Clear + */ + void Clear() + { + if (coo_tuples) delete[] coo_tuples; + coo_tuples = nullptr; + } + + + // Destructor + ~CooMatrix() + { + Clear(); + } + + + // Display matrix to stdout + void Display() + { + cout << "COO Matrix (" << num_rows << " rows, " << num_cols << " columns, " << num_nonzeros << " non-zeros):\n"; + cout << "Ordinal, Row, Column, Value\n"; + for (int i = 0; i < num_nonzeros; i++) + { + cout << '\t' << i << ',' << coo_tuples[i].row << ',' << coo_tuples[i].col << ',' << coo_tuples[i].val << "\n"; + } + } + + + /** + * Builds a symmetric COO sparse from an asymmetric CSR matrix. + */ + template + void InitCsrSymmetric(CsrMatrixT &csr_matrix) + { + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + exit(1); + } + + num_rows = csr_matrix.num_cols; + num_cols = csr_matrix.num_rows; + num_nonzeros = csr_matrix.num_nonzeros * 2; + coo_tuples = new CooTuple[num_nonzeros]; + + for (OffsetT row = 0; row < csr_matrix.num_rows; ++row) + { + for (OffsetT nonzero = csr_matrix.row_offsets[row]; nonzero < csr_matrix.row_offsets[row + 1]; ++nonzero) + { + coo_tuples[nonzero].row = row; + coo_tuples[nonzero].col = csr_matrix.column_indices[nonzero]; + coo_tuples[nonzero].val = csr_matrix.values[nonzero]; + + coo_tuples[csr_matrix.num_nonzeros + nonzero].row = coo_tuples[nonzero].col; + coo_tuples[csr_matrix.num_nonzeros + nonzero].col = coo_tuples[nonzero].row; + coo_tuples[csr_matrix.num_nonzeros + nonzero].val = csr_matrix.values[nonzero]; + + } + } + + // Sort by rows, then columns + std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); + } + + /** + * Builds a COO sparse from a relabeled CSR matrix. + */ + template + void InitCsrRelabel(CsrMatrixT &csr_matrix, const OffsetT* relabel_indices) + { + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + exit(1); + } + + num_rows = csr_matrix.num_rows; + num_cols = csr_matrix.num_cols; + num_nonzeros = csr_matrix.num_nonzeros; + coo_tuples = new CooTuple[num_nonzeros]; + + for (OffsetT row = 0; row < num_rows; ++row) + { + for (OffsetT nonzero = csr_matrix.row_offsets[row]; nonzero < csr_matrix.row_offsets[row + 1]; ++nonzero) + { + coo_tuples[nonzero].row = relabel_indices[row]; + coo_tuples[nonzero].col = relabel_indices[csr_matrix.column_indices[nonzero]]; + coo_tuples[nonzero].val = csr_matrix.values[nonzero]; + } + } + + // Sort by rows, then columns + std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); + } + + + + /** + * Builds a METIS COO sparse from the given file. + */ + void InitMetis(const string& /*metis_filename*/) const + { + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + exit(1); + } + + // TODO + } + + + /** + * Builds a MARKET COO sparse from the given file. + */ + void InitMarket( + const string& market_filename, + ValueT default_value = 1.0, + bool verbose = false) + { + if (verbose) { + printf("Reading... "); fflush(stdout); + } + + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + exit(1); + } + + std::ifstream ifs; + ifs.open(market_filename.c_str(), std::ifstream::in); + if (!ifs.good()) + { + fprintf(stderr, "Error opening file\n"); + exit(1); + } + + bool array = false; + bool symmetric = false; + bool skew = false; + int current_edge = -1; + char line[1024]; + + if (verbose) { + printf("Parsing... "); fflush(stdout); + } + + while (true) + { + ifs.getline(line, 1024); + if (!ifs.good()) + { + // Done + break; + } + + if (line[0] == '%') + { + // Comment + if (line[1] == '%') + { + // Banner + symmetric = (strstr(line, "symmetric") != nullptr); + skew = (strstr(line, "skew") != nullptr); + array = (strstr(line, "array") != nullptr); + + if (verbose) { + printf("(symmetric: %d, skew: %d, array: %d) ", symmetric, skew, array); fflush(stdout); + } + } + } + else if (current_edge == -1) + { + // Problem description + int nparsed = sscanf(line, "%d %d %d", &num_rows, &num_cols, &num_nonzeros); + if ((!array) && (nparsed == 3)) + { + if (symmetric) + num_nonzeros *= 2; + + // Allocate coo matrix + coo_tuples = new CooTuple[num_nonzeros]; + current_edge = 0; + + } + else if (array && (nparsed == 2)) + { + // Allocate coo matrix + num_nonzeros = num_rows * num_cols; + coo_tuples = new CooTuple[num_nonzeros]; + current_edge = 0; + } + else + { + fprintf(stderr, "Error parsing MARKET matrix: invalid problem description: %s\n", line); + exit(1); + } + + } + else + { + // Edge + if (current_edge >= num_nonzeros) + { + fprintf(stderr, "Error parsing MARKET matrix: encountered more than %d num_nonzeros\n", num_nonzeros); + exit(1); + } + + int row, col; + double val; + + if (array) + { + if (sscanf(line, "%lf", &val) != 1) + { + fprintf(stderr, "Error parsing MARKET matrix: badly formed current_edge: '%s' at edge %d\n", line, current_edge); + exit(1); + } + col = (current_edge / num_rows); + row = (current_edge - (num_rows * col)); + + coo_tuples[current_edge] = CooTuple(row, col, val); // Convert indices to zero-based + } + else + { + // Parse nonzero (note: using strtol and strtod is 2x faster than sscanf or istream parsing) + char *l = line; + char* t = nullptr; + + // parse row + row = strtol(l, &t, 0); + if (t == l) + { + fprintf(stderr, "Error parsing MARKET matrix: badly formed row at edge %d\n", current_edge); + exit(1); + } + l = t; + + // parse col + col = strtol(l, &t, 0); + if (t == l) + { + fprintf(stderr, "Error parsing MARKET matrix: badly formed col at edge %d\n", current_edge); + exit(1); + } + l = t; + + // parse val + val = strtod(l, &t); + if (t == l) + { + val = default_value; + } +/* + int nparsed = sscanf(line, "%d %d %lf", &row, &col, &val); + if (nparsed == 2) + { + // No value specified + val = default_value; + + } + else if (nparsed != 3) + { + fprintf(stderr, "Error parsing MARKET matrix 1: badly formed current_edge: %d parsed at edge %d\n", nparsed, current_edge); + exit(1); + } +*/ + + coo_tuples[current_edge] = CooTuple(row - 1, col - 1, val); // Convert indices to zero-based + + } + + current_edge++; + + if (symmetric && (row != col)) + { + coo_tuples[current_edge].row = coo_tuples[current_edge - 1].col; + coo_tuples[current_edge].col = coo_tuples[current_edge - 1].row; + coo_tuples[current_edge].val = coo_tuples[current_edge - 1].val * (skew ? -1 : 1); + current_edge++; + } + } + } + + // Adjust nonzero count (nonzeros along the diagonal aren't reversed) + num_nonzeros = current_edge; + + if (verbose) { + printf("done. Ordering..."); fflush(stdout); + } + + // Sort by rows, then columns + std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); + + if (verbose) { + printf("done. "); fflush(stdout); + } + + ifs.close(); + } + + + /** + * Builds a dense matrix + */ + int InitDense(OffsetT num_rows, + OffsetT num_cols, + ValueT default_value = 1.0, + bool /*verbose*/ = false) + { + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + exit(1); + } + + this->num_rows = num_rows; + this->num_cols = num_cols; + + num_nonzeros = num_rows * num_cols; + coo_tuples = new CooTuple[num_nonzeros]; + + for (OffsetT row = 0; row < num_rows; ++row) + { + for (OffsetT col = 0; col < num_cols; ++col) + { + coo_tuples[(row * num_cols) + col] = CooTuple(row, col, default_value); + } + } + + // Sort by rows, then columns + std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); + + return 0; + } + + /** + * Builds a wheel COO sparse matrix having spokes spokes. + */ + int InitWheel(OffsetT spokes, ValueT default_value = 1.0, bool /*verbose*/ = false) + { + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + exit(1); + } + + num_rows = spokes + 1; + num_cols = num_rows; + num_nonzeros = spokes * 2; + coo_tuples = new CooTuple[num_nonzeros]; + + // Add spoke num_nonzeros + int current_edge = 0; + for (OffsetT i = 0; i < spokes; i++) + { + coo_tuples[current_edge] = CooTuple(0, i + 1, default_value); + current_edge++; + } + + // Add rim + for (OffsetT i = 0; i < spokes; i++) + { + OffsetT dest = (i + 1) % spokes; + coo_tuples[current_edge] = CooTuple(i + 1, dest + 1, default_value); + current_edge++; + } + + // Sort by rows, then columns + std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); + + return 0; + } + + + /** + * Builds a square 2D grid CSR matrix. Interior num_vertices have degree 5 when including + * a self-loop. + * + * Returns 0 on success, 1 on failure. + */ + int InitGrid2d(OffsetT width, bool self_loop, ValueT default_value = 1.0) + { + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + exit(1); + } + + int interior_nodes = (width - 2) * (width - 2); + int edge_nodes = (width - 2) * 4; + int corner_nodes = 4; + num_rows = width * width; + num_cols = num_rows; + num_nonzeros = (interior_nodes * 4) + (edge_nodes * 3) + (corner_nodes * 2); + + if (self_loop) + num_nonzeros += num_rows; + + coo_tuples = new CooTuple[num_nonzeros]; + int current_edge = 0; + + for (OffsetT j = 0; j < width; j++) + { + for (OffsetT k = 0; k < width; k++) + { + OffsetT me = (j * width) + k; + + // West + OffsetT neighbor = (j * width) + (k - 1); + if (k - 1 >= 0) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // East + neighbor = (j * width) + (k + 1); + if (k + 1 < width) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // North + neighbor = ((j - 1) * width) + k; + if (j - 1 >= 0) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // South + neighbor = ((j + 1) * width) + k; + if (j + 1 < width) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + if (self_loop) + { + neighbor = me; + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + } + } + + // Sort by rows, then columns, update dims + std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); + + return 0; + } + + + /** + * Builds a square 3D grid COO sparse matrix. Interior num_vertices have degree 7 when including + * a self-loop. Values are uninitialized, coo_tuples are sorted. + */ + int InitGrid3d(OffsetT width, bool self_loop, ValueT default_value = 1.0) + { + if (coo_tuples) + { + fprintf(stderr, "Matrix already constructed\n"); + return -1; + } + + OffsetT interior_nodes = (width - 2) * (width - 2) * (width - 2); + OffsetT face_nodes = (width - 2) * (width - 2) * 6; + OffsetT edge_nodes = (width - 2) * 12; + OffsetT corner_nodes = 8; + num_cols = width * width * width; + num_rows = num_cols; + num_nonzeros = (interior_nodes * 6) + (face_nodes * 5) + (edge_nodes * 4) + (corner_nodes * 3); + + if (self_loop) + num_nonzeros += num_rows; + + coo_tuples = new CooTuple[num_nonzeros]; + int current_edge = 0; + + for (OffsetT i = 0; i < width; i++) + { + for (OffsetT j = 0; j < width; j++) + { + for (OffsetT k = 0; k < width; k++) + { + + OffsetT me = (i * width * width) + (j * width) + k; + + // Up + OffsetT neighbor = (i * width * width) + (j * width) + (k - 1); + if (k - 1 >= 0) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // Down + neighbor = (i * width * width) + (j * width) + (k + 1); + if (k + 1 < width) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // West + neighbor = (i * width * width) + ((j - 1) * width) + k; + if (j - 1 >= 0) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // East + neighbor = (i * width * width) + ((j + 1) * width) + k; + if (j + 1 < width) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // North + neighbor = ((i - 1) * width * width) + (j * width) + k; + if (i - 1 >= 0) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + // South + neighbor = ((i + 1) * width * width) + (j * width) + k; + if (i + 1 < width) { + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + + if (self_loop) + { + neighbor = me; + coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); + current_edge++; + } + } + } + } + + // Sort by rows, then columns, update dims + std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); + + return 0; + } +}; + + + +/****************************************************************************** + * COO matrix type + ******************************************************************************/ + + +/** + * CSR sparse format matrix + */ +template< + typename ValueT, + typename OffsetT> +struct CsrMatrix +{ + int num_rows; + int num_cols; + int num_nonzeros; + OffsetT* row_offsets; + OffsetT* column_indices; + ValueT* values; + bool numa_malloc; + + /** + * Constructor + */ + CsrMatrix() + : num_rows(0) + , num_cols(0) + , num_nonzeros(0) + , row_offsets(nullptr) + , column_indices(nullptr) + , values(nullptr) + { +#ifdef CUB_MKL + numa_malloc = ((numa_available() >= 0) && (numa_num_task_nodes() > 1)); +#else + numa_malloc = false; +#endif + } + + + /** + * Clear + */ + void Clear() + { +#ifdef CUB_MKL + if (numa_malloc) + { + numa_free(row_offsets, sizeof(OffsetT) * (num_rows + 1)); + numa_free(values, sizeof(ValueT) * num_nonzeros); + numa_free(column_indices, sizeof(OffsetT) * num_nonzeros); + } + else + { + if (row_offsets) mkl_free(row_offsets); + if (column_indices) mkl_free(column_indices); + if (values) mkl_free(values); + } + +#else + if (row_offsets) delete[] row_offsets; + if (column_indices) delete[] column_indices; + if (values) delete[] values; +#endif + + row_offsets = nullptr; + column_indices = nullptr; + values = nullptr; + } + + /** + * Destructor + */ + ~CsrMatrix() + { + Clear(); + } + + GraphStats Stats() const + { + GraphStats stats; + stats.num_rows = num_rows; + stats.num_cols = num_cols; + stats.num_nonzeros = num_nonzeros; + + // + // Compute diag-distance statistics + // + + OffsetT samples = 0; + double mean = 0.0; + double ss_tot = 0.0; + + for (OffsetT row = 0; row < num_rows; ++row) + { + OffsetT nz_idx_start = row_offsets[row]; + OffsetT nz_idx_end = row_offsets[row + 1]; + + for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx) + { + OffsetT col = column_indices[nz_idx]; + double x = (col > row) ? col - row : row - col; + + samples++; + double delta = x - mean; + mean = mean + (delta / samples); + ss_tot += delta * (x - mean); + } + } + stats.diag_dist_mean = mean; + double variance = ss_tot / samples; + stats.diag_dist_std_dev = sqrt(variance); + + + // + // Compute deming statistics + // + + samples = 0; + double mean_x = 0.0; + double mean_y = 0.0; + double ss_x = 0.0; + double ss_y = 0.0; + + for (OffsetT row = 0; row < num_rows; ++row) + { + OffsetT nz_idx_start = row_offsets[row]; + OffsetT nz_idx_end = row_offsets[row + 1]; + + for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx) + { + OffsetT col = column_indices[nz_idx]; + + samples++; + double x = col; + double y = row; + double delta; + + delta = x - mean_x; + mean_x = mean_x + (delta / samples); + ss_x += delta * (x - mean_x); + + delta = y - mean_y; + mean_y = mean_y + (delta / samples); + ss_y += delta * (y - mean_y); + } + } + + samples = 0; + double s_xy = 0.0; + double s_xxy = 0.0; + double s_xyy = 0.0; + for (OffsetT row = 0; row < num_rows; ++row) + { + OffsetT nz_idx_start = row_offsets[row]; + OffsetT nz_idx_end = row_offsets[row + 1]; + + for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx) + { + OffsetT col = column_indices[nz_idx]; + + samples++; + double x = col; + double y = row; + + double xy = (x - mean_x) * (y - mean_y); + double xxy = (x - mean_x) * (x - mean_x) * (y - mean_y); + double xyy = (x - mean_x) * (y - mean_y) * (y - mean_y); + double delta; + + delta = xy - s_xy; + s_xy = s_xy + (delta / samples); + + delta = xxy - s_xxy; + s_xxy = s_xxy + (delta / samples); + + delta = xyy - s_xyy; + s_xyy = s_xyy + (delta / samples); + } + } + + // double s_xx = ss_x / num_nonzeros; + // double s_yy = ss_y / num_nonzeros; + + stats.pearson_r = (num_nonzeros * s_xy) / (sqrt(ss_x) * sqrt(ss_y)); + + + // + // Compute row-length statistics + // + + // Sample mean + stats.row_length_mean = double(num_nonzeros) / num_rows; + variance = 0.0; + stats.row_length_skewness = 0.0; + for (OffsetT row = 0; row < num_rows; ++row) + { + OffsetT length = row_offsets[row + 1] - row_offsets[row]; + double delta = double(length) - stats.row_length_mean; + variance += (delta * delta); + stats.row_length_skewness += (delta * delta * delta); + } + variance /= num_rows; + stats.row_length_std_dev = sqrt(variance); + stats.row_length_skewness = (stats.row_length_skewness / num_rows) / pow(stats.row_length_std_dev, 3.0); + stats.row_length_variation = stats.row_length_std_dev / stats.row_length_mean; + + return stats; + } + + /** + * Build CSR matrix from sorted COO matrix + */ + void FromCoo(const CooMatrix &coo_matrix) + { + num_rows = coo_matrix.num_rows; + num_cols = coo_matrix.num_cols; + num_nonzeros = coo_matrix.num_nonzeros; + +#ifdef CUB_MKL + + if (numa_malloc) + { + numa_set_strict(1); +// numa_set_bind_policy(1); + +// values = (ValueT*) numa_alloc_interleaved(sizeof(ValueT) * num_nonzeros); +// row_offsets = (OffsetT*) numa_alloc_interleaved(sizeof(OffsetT) * (num_rows + 1)); +// column_indices = (OffsetT*) numa_alloc_interleaved(sizeof(OffsetT) * num_nonzeros); + + row_offsets = (OffsetT*) numa_alloc_onnode(sizeof(OffsetT) * (num_rows + 1), 0); + column_indices = (OffsetT*) numa_alloc_onnode(sizeof(OffsetT) * num_nonzeros, 0); + values = (ValueT*) numa_alloc_onnode(sizeof(ValueT) * num_nonzeros, 1); + } + else + { + values = (ValueT*) mkl_malloc(sizeof(ValueT) * num_nonzeros, 4096); + row_offsets = (OffsetT*) mkl_malloc(sizeof(OffsetT) * (num_rows + 1), 4096); + column_indices = (OffsetT*) mkl_malloc(sizeof(OffsetT) * num_nonzeros, 4096); + + } + +#else + row_offsets = new OffsetT[num_rows + 1]; + column_indices = new OffsetT[num_nonzeros]; + values = new ValueT[num_nonzeros]; +#endif + + OffsetT prev_row = -1; + for (OffsetT current_edge = 0; current_edge < num_nonzeros; current_edge++) + { + OffsetT current_row = coo_matrix.coo_tuples[current_edge].row; + + // Fill in rows up to and including the current row + for (OffsetT row = prev_row + 1; row <= current_row; row++) + { + row_offsets[row] = current_edge; + } + prev_row = current_row; + + column_indices[current_edge] = coo_matrix.coo_tuples[current_edge].col; + values[current_edge] = coo_matrix.coo_tuples[current_edge].val; + } + + // Fill out any trailing edgeless vertices (and the end-of-list element) + for (OffsetT row = prev_row + 1; row <= num_rows; row++) + { + row_offsets[row] = num_nonzeros; + } + } + + + /** + * Display log-histogram to stdout + */ + void DisplayHistogram() + { + // Initialize + int log_counts[9]; + for (int i = 0; i < 9; i++) + { + log_counts[i] = 0; + } + + // Scan + int max_log_length = -1; + for (OffsetT row = 0; row < num_rows; row++) + { + OffsetT length = row_offsets[row + 1] - row_offsets[row]; + + int log_length = -1; + while (length > 0) + { + length /= 10; + log_length++; + } + if (log_length > max_log_length) + { + max_log_length = log_length; + } + + log_counts[log_length + 1]++; + } + printf("CSR matrix (%d rows, %d columns, %d non-zeros):\n", (int) num_rows, (int) num_cols, (int) num_nonzeros); + for (int i = -1; i < max_log_length + 1; i++) + { + printf("\tDegree 1e%d: \t%d (%.2f%%)\n", i, log_counts[i + 1], (float) log_counts[i + 1] * 100.0 / num_cols); + } + fflush(stdout); + } + + + /** + * Display matrix to stdout + */ + void Display() + { + printf("Input Matrix:\n"); + for (OffsetT row = 0; row < num_rows; row++) + { + printf("%d [@%d, #%d]: ", row, row_offsets[row], row_offsets[row + 1] - row_offsets[row]); + for (OffsetT current_edge = row_offsets[row]; current_edge < row_offsets[row + 1]; current_edge++) + { + printf("%d (%f), ", column_indices[current_edge], values[current_edge]); + } + printf("\n"); + } + fflush(stdout); + } + + +}; + + + +/****************************************************************************** + * Matrix transformations + ******************************************************************************/ + +// Comparator for ordering rows by degree (lowest first), then by row-id (lowest first) +template +struct OrderByLow +{ + OffsetT* row_degrees; + OrderByLow(OffsetT* row_degrees) : row_degrees(row_degrees) {} + + bool operator()(const OffsetT &a, const OffsetT &b) const + { + if (row_degrees[a] < row_degrees[b]) + return true; + else if (row_degrees[a] > row_degrees[b]) + return false; + else + return (a < b); + } +}; + +// Comparator for ordering rows by degree (highest first), then by row-id (lowest first) +template +struct OrderByHigh +{ + OffsetT* row_degrees; + OrderByHigh(OffsetT* row_degrees) : row_degrees(row_degrees) {} + + bool operator()(const OffsetT &a, const OffsetT &b) const + { + if (row_degrees[a] > row_degrees[b]) + return true; + else if (row_degrees[a] < row_degrees[b]) + return false; + else + return (a < b); + } +}; + + + +/** + * Reverse Cuthill-McKee + */ +template +void RcmRelabel( + CsrMatrix& matrix, + OffsetT* relabel_indices) +{ + // Initialize row degrees + OffsetT* row_degrees_in = new OffsetT[matrix.num_rows]; + OffsetT* row_degrees_out = new OffsetT[matrix.num_rows]; + for (OffsetT row = 0; row < matrix.num_rows; ++row) + { + row_degrees_in[row] = 0; + row_degrees_out[row] = matrix.row_offsets[row + 1] - matrix.row_offsets[row]; + } + for (OffsetT nonzero = 0; nonzero < matrix.num_nonzeros; ++nonzero) + { + row_degrees_in[matrix.column_indices[nonzero]]++; + } + + // Initialize unlabeled set + using UnlabeledSet = std::set>; + typename UnlabeledSet::key_compare unlabeled_comp(row_degrees_in); + UnlabeledSet unlabeled(unlabeled_comp); + for (OffsetT row = 0; row < matrix.num_rows; ++row) + { + relabel_indices[row] = -1; + unlabeled.insert(row); + } + + // Initialize queue set + std::deque q; + + // Process unlabeled vertices (traverse connected components) + OffsetT relabel_idx = 0; + while (!unlabeled.empty()) + { + // Seed the unvisited frontier queue with the unlabeled vertex of lowest-degree + OffsetT vertex = *unlabeled.begin(); + q.push_back(vertex); + + while (!q.empty()) + { + vertex = q.front(); + q.pop_front(); + + if (relabel_indices[vertex] == -1) + { + // Update this vertex + unlabeled.erase(vertex); + relabel_indices[vertex] = relabel_idx; + relabel_idx++; + + // Sort neighbors by degree + OrderByLow neighbor_comp(row_degrees_in); + std::sort( + matrix.column_indices + matrix.row_offsets[vertex], + matrix.column_indices + matrix.row_offsets[vertex + 1], + neighbor_comp); + + // Inspect neighbors, adding to the out frontier if unlabeled + for (OffsetT neighbor_idx = matrix.row_offsets[vertex]; + neighbor_idx < matrix.row_offsets[vertex + 1]; + ++neighbor_idx) + { + OffsetT neighbor = matrix.column_indices[neighbor_idx]; + q.push_back(neighbor); + } + } + } + } + +/* + // Reverse labels + for (int row = 0; row < matrix.num_rows; ++row) + { + relabel_indices[row] = matrix.num_rows - relabel_indices[row] - 1; + } +*/ + + // Cleanup + if (row_degrees_in) delete[] row_degrees_in; + if (row_degrees_out) delete[] row_degrees_out; +} + + +/** + * Reverse Cuthill-McKee + */ +template +void RcmRelabel( + CsrMatrix& matrix, + bool verbose = false) +{ + // Do not process if not square + if (matrix.num_cols != matrix.num_rows) + { + if (verbose) { + printf("RCM transformation ignored (not square)\n"); fflush(stdout); + } + return; + } + + // Initialize relabel indices + OffsetT* relabel_indices = new OffsetT[matrix.num_rows]; + + if (verbose) { + printf("RCM relabeling... "); fflush(stdout); + } + + RcmRelabel(matrix, relabel_indices); + + if (verbose) { + printf("done. Reconstituting... "); fflush(stdout); + } + + // Create a COO matrix from the relabel indices + CooMatrix coo_matrix; + coo_matrix.InitCsrRelabel(matrix, relabel_indices); + + // Reconstitute the CSR matrix from the sorted COO tuples + if (relabel_indices) delete[] relabel_indices; + matrix.Clear(); + matrix.FromCoo(coo_matrix); + + if (verbose) { + printf("done. "); fflush(stdout); + } +} diff --git a/projects/hipcub/test/hipcub/half.hpp b/projects/hipcub/test/hipcub/half.hpp index 82b78a6096f5..f1fa31eebcd0 100644 --- a/projects/hipcub/test/hipcub/half.hpp +++ b/projects/hipcub/test/hipcub/half.hpp @@ -1,7 +1,6 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,11 +38,7 @@ #include #if defined(__HIP_PLATFORM_NVIDIA__) - #include - #include #include -#else - #include #endif #include @@ -55,29 +50,6 @@ #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif -struct half_t; - -#if defined(__HIP_PLATFORM_NVIDIA__) - -namespace cuda -{ -namespace std -{ -template<> -struct is_floating_point : true_type -{}; - -template<> -class numeric_limits -{ -public: - static constexpr bool is_specialized = true; -}; - -} // namespace std -} // namespace cuda - -#endif // __HIP_PLATFORM_NVIDIA__ /****************************************************************************** * half_t @@ -103,20 +75,18 @@ struct half_t *this = half_t(float(a)); } - /// Constructor from size_t - __host__ __device__ __forceinline__ - half_t(size_t a) + /// Constructor from std::size_t + __host__ __device__ __forceinline__ half_t(std::size_t a) { *this = half_t(float(a)); } /// Constructor from unsigned long long int template - && (!std::is_same_v)>::type> - __host__ __device__ __forceinline__ - half_t(T a) + typename = typename std::enable_if< + std::is_same::value + && (!std::is_same::value)>::type> + __host__ __device__ __forceinline__ half_t(T a) { *this = half_t(float(a)); } @@ -228,7 +198,7 @@ struct half_t f = (0xff << 23) | (sign << 31); // inf } } - static_assert(sizeof(float) == sizeof(uint32_t), "4-byte size check"); + static_assert(sizeof(float) == sizeof(std::uint32_t), "4-byte size check"); float ret{}; std::memcpy(&ret, &f, sizeof(float)); return ret; @@ -361,10 +331,25 @@ inline std::ostream& operator<<(std::ostream &out, const half_t &x) * Traits overloads ******************************************************************************/ -template<> -struct hipcub::NumericTraits - : hipcub::BaseTraits -{}; +template <> +struct hipcub::FpLimits +{ + static __host__ __device__ __forceinline__ half_t Max() { return half_t::max(); } + + static __host__ __device__ __forceinline__ half_t Lowest() { return half_t::lowest(); } +}; + +#if defined(__HIP_PLATFORM_NVIDIA__) +_CCCL_SUPPRESS_DEPRECATED_PUSH +#else +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +#endif +template <> struct hipcub::NumericTraits : hipcub::BaseTraits {}; +#if defined(__HIP_PLATFORM_NVIDIA__) +_CCCL_SUPPRESS_DEPRECATED_POP +#else +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +#endif #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/projects/hipcub/test/hipcub/single_index_iterator.hpp b/projects/hipcub/test/hipcub/single_index_iterator.hpp index b33c73c54956..8ba95e600d2f 100644 --- a/projects/hipcub/test/hipcub/single_index_iterator.hpp +++ b/projects/hipcub/test/hipcub/single_index_iterator.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -50,13 +50,6 @@ class single_index_iterator return *this; } - // Implicit conversion for read access - HIPCUB_HOST_DEVICE - inline operator T() const - { - return *value_; - } - private: T* const value_; const bool keep_; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp index 65a43392fe7f..43ce5be89a46 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,10 +23,11 @@ #include "common_test_header.hpp" // required rocprim headers +#include #include #include #include -#include +#include template< class T, @@ -71,25 +72,25 @@ struct custom_op2 }; using ParamsSubtract - = ::testing::Types, + = ::testing::Types, params_subtract, params_subtract, params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract>; @@ -253,7 +254,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeft) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t, int, output_type>; + using stored_type = std::conditional_t::value, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -336,8 +337,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeft) ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, - _HIPCUB_STD::max(test_utils::precision::value, - test_utils::precision::value))); + std::max(test_utils::precision::value, + test_utils::precision::value))); HIP_CHECK(hipFree(d_input)); HIP_CHECK(hipFree(d_output)); @@ -355,7 +356,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeftPartialTile) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t, int, output_type>; + using stored_type = std::conditional_t::value, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -457,8 +458,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeftPartialTile) ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, - _HIPCUB_STD::max(test_utils::precision::value, - test_utils::precision::value))); + std::max(test_utils::precision::value, + test_utils::precision::value))); HIP_CHECK(hipFree(d_input)); HIP_CHECK(hipFree(d_tile_sizes)); @@ -477,7 +478,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRight) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t, int, output_type>; + using stored_type = std::conditional_t::value, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -560,8 +561,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRight) ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, - _HIPCUB_STD::max(test_utils::precision::value, - test_utils::precision::value))); + std::max(test_utils::precision::value, + test_utils::precision::value))); HIP_CHECK(hipFree(d_input)); HIP_CHECK(hipFree(d_output)); @@ -579,7 +580,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRightPartialTile) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t, int, output_type>; + using stored_type = std::conditional_t::value, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -681,8 +682,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRightPartialTile) // clang-format off ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(output, expected, is_add_op::value - ? _HIPCUB_STD::max(test_utils::precision::value, test_utils::precision::value) - : std::is_same_v + ? std::max(test_utils::precision::value, test_utils::precision::value) + : std::is_same::value ? 0 : test_utils::precision::value)); // clang-format on diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp index 61e44a99023e..0bed699bdefb 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ #include #include #include +#include template struct params @@ -72,8 +73,8 @@ bool apply(FlagOp flag_op, const T& a, const T& b, unsigned int) using Params = ::testing::Types< // Power of 2 BlockSize - params, - params, + params, + params, params, params, params, @@ -86,18 +87,18 @@ using Params = ::testing::Types< params, params, params, - params, - params, + params, + params, // Power of 2 BlockSize and ItemsPerThread > 1 params>, params, params>, - params, + params, // Non-power of 2 BlockSize and ItemsPerThread > 1 params>, - params, + params, params, params, params, @@ -145,10 +146,10 @@ TYPED_TEST(HipcubBlockDiscontinuity, FlagHeads) using type = typename TestFixture::params::type; // std::vector is a special case that will cause an error in hipMemcpy - using stored_flag_type = - typename std::conditional, - int, - typename TestFixture::params::flag_type>::type; + using stored_flag_type = typename std::conditional< + std::is_same::value, + int, + typename TestFixture::params::flag_type>::type; using flag_type = typename TestFixture::params::flag_type; using flag_op_type = typename TestFixture::params::flag_op_type; constexpr size_t block_size = TestFixture::params::block_size; @@ -283,10 +284,10 @@ TYPED_TEST(HipcubBlockDiscontinuity, FlagTails) using type = typename TestFixture::params::type; // std::vector is a special case that will cause an error in hipMemcpy - using stored_flag_type = - typename std::conditional, - int, - typename TestFixture::params::flag_type>::type; + using stored_flag_type = typename std::conditional< + std::is_same::value, + int, + typename TestFixture::params::flag_type>::type; using flag_type = typename TestFixture::params::flag_type; using flag_op_type = typename TestFixture::params::flag_op_type; constexpr size_t block_size = TestFixture::params::block_size; @@ -450,10 +451,10 @@ TYPED_TEST(HipcubBlockDiscontinuity, FlagHeadsAndTails) using type = typename TestFixture::params::type; // std::vector is a special case that will cause an error in hipMemcpy - using stored_flag_type = - typename std::conditional, - int, - typename TestFixture::params::flag_type>::type; + using stored_flag_type = typename std::conditional< + std::is_same::value, + int, + typename TestFixture::params::flag_type>::type; using flag_type = typename TestFixture::params::flag_type; using flag_op_type = typename TestFixture::params::flag_op_type; constexpr size_t block_size = TestFixture::params::block_size; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp index 4db4d9ea1370..afa6f4c6265d 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -908,7 +908,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedGuarded) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 < size) + if(i1 >= 0 && i1 < size) host_expected[i1] = host_input[i0]; } } @@ -1025,7 +1025,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedFlagged) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 < size) + if(i1 >= 0 && i1 < size) host_expected[i1] = host_input[i0]; host_flags[i0] = (ti == block_size - 1) && (ii == items_per_thread - 1) ? false : true; @@ -1818,7 +1818,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedGuardedNoOutputParam) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 < size) + if(i1 >= 0 && i1 < size) host_expected[i1] = host_input[i0]; } } @@ -1930,7 +1930,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedFlaggedNoOutputParam) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 < size) + if(i1 >= 0 && i1 < size) host_expected[i1] = host_input[i0]; host_flags[i0] = (ti == block_size - 1) && (ii == items_per_thread - 1) ? false : true; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp index 78f55bdd6264..0691aabde326 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -117,7 +117,7 @@ void histogram_kernel(T* device_output, T* device_output_bin) bhistogram_t(temp_storage).Histogram(in_out, hist); __syncthreads(); - _CCCL_PRAGMA_UNROLL_FULL() + #pragma unroll for (unsigned int offset = 0; offset < BinSize; offset += BlockSize) { if(offset + hipThreadIdx_x < BinSize) diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp index 04fbb2b522d4..b9ff9334b3f8 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2020, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,6 +34,7 @@ // kernel definitions #include "test_hipcub_block_load_store.kernels.hpp" +#include // Start stamping out tests struct HipcubBlockLoadStoreTests; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp index bd39aeb70d9c..bf5fa0e2e1cb 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -371,19 +371,24 @@ typed_test_def(HipcubBlockLoadStoreTests, name_suffix, LoadStoreDiscardIterator) input.size() * sizeof(typename decltype(input)::value_type), hipMemcpyHostToDevice)); - // Running kernel for discard case - Type* dummy; - HIP_CHECK(hipMalloc(&dummy, guarded_elements * sizeof(Type))); + // Test with discard output iterator + // using OffsetT = typename std::iterator_traits::difference_type; + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + // TODO: Here in block load an store, it's not possible to use rocprim::discard_iterator + hipcub::DiscardOutputIterator discard_itr; - load_store_guarded_kernel, load_method, store_method, block_size, items_per_thread> - <<>>(device_input, dummy, dummy, guarded_elements); - HIP_CHECK(hipFree(dummy)); - + <<>>(device_input, + discard_itr, + discard_itr, + guarded_elements); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP // Running kernel load_store_guarded_kernel; + using InputT = typename std::iterator_traits::value_type; // The output value type using OutputT = typename std::conditional< - (std::is_same_v, - void>), // OutputT = (if output iterator's value type is void) ? - hipcub::detail::it_value_t, // ... then the input iterator's + (std::is_same::value_type, + void>::value), // OutputT = (if output iterator's value type is void) ? + typename std::iterator_traits::value_type, // ... then the input iterator's // value type, - hipcub::detail::it_value_t>:: + typename std::iterator_traits::value_type>:: type; // ... else the output iterator's value type // Threadblock load/store abstraction types @@ -191,7 +191,7 @@ __launch_bounds__(BlockSize) __global__ __syncthreads(); // reset data - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int item = 0; item < ItemsPerThread; ++item) data[item] = OutputT(); diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp index 0258a33880e3..6eda70ee2534 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -123,11 +123,10 @@ TYPED_TEST(HipcubBlockMergeSort, SortKeys) // Generate data std::vector keys_output; - keys_output - = test_utils::get_random_data(size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + keys_output = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value); // Calculate expected results on host std::vector expected(keys_output); @@ -183,45 +182,19 @@ void sort_key_with_valid_items_kernel(T* device_input, T default_val) { constexpr size_t items_per_block = items_per_thread * block_size; - - const int block_offset = static_cast(blockIdx.x * items_per_block); - const int thread_offset = static_cast(threadIdx.x * items_per_thread); + const size_t offset = (blockIdx.x * items_per_block) + (threadIdx.x * items_per_thread); T input[items_per_thread]; - // Define per-thread valid range within the block - const int thread_start = thread_offset; - const int thread_end = thread_start + static_cast(items_per_thread); - - // Count valid items this thread actually owns - const int local_valid = (thread_start >= valid_items) - ? 0 - : ((thread_end <= valid_items) ? static_cast(items_per_thread) - : (valid_items - thread_start)); - - // Load valid items and fill the rest with default_val for(size_t i = 0; i < items_per_thread; i++) - { - const int idx = block_offset + thread_offset + static_cast(i); - if(static_cast(i) < local_valid) - input[i] = device_input[idx]; - else - input[i] = default_val; - } + input[i] = device_input[offset + i]; - using BlockSort = hipcub::BlockMergeSort; - __shared__ - typename BlockSort::TempStorage temp_storage; - BlockSort bsort(temp_storage); + hipcub::BlockMergeSort bsort; - // Sort the whole block since all invalid items are already default_val - bsort.Sort(input, compare_op); + bsort.Sort(input, compare_op, valid_items, default_val); for(size_t i = 0; i < items_per_thread; i++) - { - const int idx = block_offset + thread_offset + static_cast(i); - device_input[idx] = input[i]; - } + device_input[offset + i] = input[i]; } TYPED_TEST(HipcubBlockMergeSort, SortKeysWithValidItems) @@ -244,8 +217,8 @@ TYPED_TEST(HipcubBlockMergeSort, SortKeysWithValidItems) constexpr size_t size = grid_size * items_per_block; // minus|plus two to prevent overflow weirdness - const T mini = _HIPCUB_STD::numeric_limits::min() + static_cast(2); - const T maxi = _HIPCUB_STD::numeric_limits::max() - static_cast(2); + const T mini = std::numeric_limits::min() + static_cast(2); + const T maxi = std::numeric_limits::max() - static_cast(2); const T default_val = static_cast(compare_op(mini, maxi) ? maxi : mini); const int valid_items_arr[8] = {items_per_block / 2, @@ -381,18 +354,17 @@ TYPED_TEST(HipcubBlockMergeSort, SortKeysValues) // Generate data std::vector keys_output; - keys_output - = test_utils::get_random_data(size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + keys_output = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value); std::vector values_output; - values_output = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + values_output + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); using key_value = std::pair; @@ -621,18 +593,17 @@ TYPED_TEST(HipcubBlockMergeSort, StableSortKeysValues) // Generate data std::vector keys_output; - keys_output - = test_utils::get_random_data(size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + keys_output = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value); std::vector values_output; - values_output = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + values_output + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); // Set some keys to be the same, but have different values to test stability for(size_t i = 0; i < 10; i++) @@ -723,45 +694,23 @@ void stable_sort_key_with_valid_items_kernel(T* device_input, T default_val) { constexpr size_t items_per_block = items_per_thread * block_size; - const int block_offset = static_cast(blockIdx.x * items_per_block); - const int thread_offset = static_cast(threadIdx.x * items_per_thread); + const size_t offset = (blockIdx.x * items_per_block) + (threadIdx.x * items_per_thread); T input[items_per_thread]; - // Define per-thread valid range within the block - const int thread_start = thread_offset; - const int thread_end = thread_start + static_cast(items_per_thread); - - const int local_valid = (thread_start >= valid_items) - ? 0 - : ((thread_end <= valid_items) ? static_cast(items_per_thread) - : (valid_items - thread_start)); - - // Load valid items and fill invalid ones with default_val for(size_t i = 0; i < items_per_thread; i++) - { - const int idx = block_offset + thread_offset + static_cast(i); - - if(static_cast(i) < local_valid) - input[i] = device_input[idx]; - else - input[i] = default_val; - } + input[i] = device_input[offset + i]; - using BlockSort = hipcub::BlockMergeSort; - __shared__ - typename BlockSort::TempStorage temp_storage; - BlockSort bsort(temp_storage); + hipcub::BlockMergeSort bsort; - // Stable-sort the whole block since all invalid items are masked - bsort.StableSort(input, - [&](const T& lhs, const T& rhs) { return compare_op(lhs.elem, rhs.elem); }); + bsort.StableSort( + input, + [&](const T& lhs, const T& rhs) { return compare_op(lhs.elem, rhs.elem); }, + valid_items, + default_val); for(size_t i = 0; i < items_per_thread; i++) - { - const int idx = block_offset + thread_offset + static_cast(i); - device_input[idx] = input[i]; - } + device_input[offset + i] = input[i]; } TYPED_TEST(HipcubBlockMergeSort, StableSortKeysWithValidItems) @@ -785,8 +734,8 @@ TYPED_TEST(HipcubBlockMergeSort, StableSortKeysWithValidItems) constexpr size_t size = grid_size * items_per_block; // minus|plus two to prevent overflow weirdness - const T mini = _HIPCUB_STD::numeric_limits::min() + static_cast(2); - const T maxi = _HIPCUB_STD::numeric_limits::max() - static_cast(2); + const T mini = std::numeric_limits::min() + static_cast(2); + const T maxi = std::numeric_limits::max() - static_cast(2); const custom_type default_val = {static_cast(compare_op(mini, maxi) ? maxi : mini), 0}; const int valid_items_arr[8] = {items_per_block / 2, @@ -885,52 +834,25 @@ void stable_sort_key_value_with_valid_items_kernel(T* device_key_input, T default_val) { constexpr size_t items_per_block = items_per_thread * block_size; - - const int block_offset = static_cast(blockIdx.x * items_per_block); - const int thread_offset = static_cast(threadIdx.x * items_per_thread); + const size_t offset = (blockIdx.x * items_per_block) + (threadIdx.x * items_per_thread); T key_input[items_per_thread]; T value_input[items_per_thread]; - // Define per-thread valid range - const int thread_start = thread_offset; - const int thread_end = thread_start + static_cast(items_per_thread); - - const int local_valid = (thread_start >= valid_items) - ? 0 - : ((thread_end <= valid_items) ? static_cast(items_per_thread) - : (valid_items - thread_start)); - - // Load valid items and fill invalid ones with default_val for(size_t i = 0; i < items_per_thread; i++) { - const int idx = block_offset + thread_offset + static_cast(i); - - if(static_cast(i) < local_valid) - { - key_input[i] = device_key_input[idx]; - value_input[i] = device_value_input[idx]; - } - else - { - key_input[i] = default_val; - value_input[i] = device_value_input[idx]; - } + key_input[i] = device_key_input[offset + i]; + value_input[i] = device_value_input[offset + i]; } - using BlockSort = hipcub::BlockMergeSort; - __shared__ - typename BlockSort::TempStorage temp_storage; - BlockSort bsort(temp_storage); + hipcub::BlockMergeSort bsort; - // Sort entire block since all invalid items are masked - bsort.StableSort(key_input, value_input, compare_op); + bsort.StableSort(key_input, value_input, compare_op, valid_items, default_val); for(size_t i = 0; i < items_per_thread; i++) { - const int idx = block_offset + thread_offset + static_cast(i); - device_key_input[idx] = key_input[i]; - device_value_input[idx] = value_input[i]; + device_key_input[offset + i] = key_input[i]; + device_value_input[offset + i] = value_input[i]; } } @@ -960,8 +882,8 @@ TYPED_TEST(HipcubBlockMergeSort, StableSortKeysValuesWithValidItems) constexpr size_t size = grid_size * items_per_block; // minus|plus two to prevent overflow weirdness - const T mini = _HIPCUB_STD::numeric_limits::min() + static_cast(2); - const T maxi = _HIPCUB_STD::numeric_limits::max() - static_cast(2); + const T mini = std::numeric_limits::min() + static_cast(2); + const T maxi = std::numeric_limits::max() - static_cast(2); T default_val = static_cast(compare_op(mini, maxi) ? maxi : mini); const int valid_items_arr[8] = {items_per_block / 2, diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp index f8311dfb6beb..9e5ca6ccb38b 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp @@ -341,7 +341,7 @@ void rank_kernel(const KeyType* keys_input, UnsignedBits(&unsigned_keys)[ItemsPerThread] = reinterpret_cast(keys); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int key = 0; key < ItemsPerThread; key++) { unsigned_keys[key] = KeyTraits::TwiddleIn(unsigned_keys[key]); @@ -531,39 +531,18 @@ void rank_with_prefix_sum_kernel(const KeyType* keys_input, int* prefix_sum_output, unsigned int start_bit) { -#if defined(__HIP_PLATFORM_NVIDIA__) - constexpr bool warp_striped = false; // CUB BlockRadixRankMatch expects blocked layout -#else constexpr bool warp_striped = Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH; -#endif using KeyTraits = hipcub::Traits; using UnsignedBits = typename KeyTraits::UnsignedBits; using DigitExtractor = hipcub::BFEDigitExtractor; - - using RankType = -#if defined(__HIP_PLATFORM_NVIDIA__) - // For CUB + ULL + MATCH, fall back to basic BlockRadixRank - std::conditional_t< - Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH - && std::is_same_v, - hipcub::BlockRadixRank, - std::conditional_t< - Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH, - hipcub::BlockRadixRankMatch, - hipcub::BlockRadixRank>>; -#else - std::conditional_t< - Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH, - hipcub::BlockRadixRankMatch, - hipcub::BlockRadixRank>; -#endif + using RankType = std::conditional_t< + Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH, + hipcub::BlockRadixRankMatch, + hipcub::BlockRadixRank>; using KeyExchangeType = hipcub::BlockExchange; using RankExchangeType = hipcub::BlockExchange; @@ -592,7 +571,7 @@ void rank_with_prefix_sum_kernel(const KeyType* keys_input, UnsignedBits(&unsigned_keys)[ItemsPerThread] = reinterpret_cast(keys); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(unsigned int key = 0; key < ItemsPerThread; key++) { unsigned_keys[key] = KeyTraits::TwiddleIn(unsigned_keys[key]); @@ -616,28 +595,13 @@ void rank_with_prefix_sum_kernel(const KeyType* keys_input, hipcub::StoreDirectBlocked(lid, ranks_output + block_offset, ranks); - const size_t pfs_size = (1 << RadixBits); - const size_t pfs_offset = (blockIdx.x * pfs_size); + const size_t pfs_size = (1 << RadixBits); + const size_t pfs_offset = (blockIdx.x * pfs_size) + (threadIdx.x * bins_tracked_per_thread); for(size_t i = 0; i < bins_tracked_per_thread; i++) { - const size_t local_bin = threadIdx.x * bins_tracked_per_thread + i; - if(local_bin >= pfs_size) - continue; - -#if defined(__HIP_PLATFORM_NVIDIA__) - if constexpr(std::is_same_v && Descending) - { - // Make CUB's layout match rocPRIM's: flip the global bin index - const size_t mirrored_bin = pfs_size - 1 - local_bin; - prefix_sum_output[pfs_offset + mirrored_bin] = prefix_sum_storage[i]; - } - else -#endif - { - // Normal (rocPRIM-compatible) layout - prefix_sum_output[pfs_offset + local_bin] = prefix_sum_storage[i]; - } + if((threadIdx.x * bins_tracked_per_thread) + i < pfs_size) + prefix_sum_output[pfs_offset + i] = prefix_sum_storage[i]; } } @@ -676,7 +640,7 @@ void test_radix_rank_with_prefix_sum_output() constexpr unsigned end_bit = start_bit + radix_bits; constexpr size_t items_per_block = block_size * items_per_thread; - if constexpr(std::is_same_v) + if constexpr(std::is_same::value) { // Given block size not supported @@ -751,10 +715,10 @@ void test_radix_rank_with_prefix_sum_output() uint64_t bit_rep = c.out; bit_rep >>= start_bit; - bit_rep &= ((1ull << radix_bits) - 1); + bit_rep &= ((1 << radix_bits) - 1); if(descending) - bit_rep = (1ull << radix_bits) - (1 + bit_rep); //flip it + bit_rep = (1 << radix_bits) - (1 + bit_rep); //flip it ++histogram[bit_rep]; } @@ -819,22 +783,14 @@ void test_radix_rank_with_prefix_sum_output() { SCOPED_TRACE(testing::Message() << "with index= " << i); ASSERT_EQ(ranks_output[i], expected[i]); + + if(i < pfs_size) + ASSERT_EQ(prefix_sum_output[i], pfs_expected[i]); } HIP_CHECK(hipFree(d_keys_input)); HIP_CHECK(hipFree(d_ranks_output)); HIP_CHECK(hipFree(d_prefix_sum_output)); - - for(size_t block = 0; block < grid_size; ++block) - { - const size_t block_pfs_offset = block * pfs_items_per_block; - - for(size_t bin = 0; bin < pfs_items_per_block; ++bin) - { - const size_t idx = block_pfs_offset + bin; - ASSERT_EQ(prefix_sum_output[idx], pfs_expected[idx]); - } - } } } } diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp index 7a156732f9a3..f5abe06154b2 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -60,7 +60,7 @@ class HipcubBlockRadixSort : public ::testing::Test using Params = ::testing::Types< // Power of 2 BlockSize -#if _CCCL_HAS_INT128() +#if HIPCUB_IS_INT128_ENABLED params<__int128_t, __int128_t, 64U, 1>, params<__uint128_t, __uint128_t, 64U, 1>, #endif @@ -448,11 +448,11 @@ TYPED_TEST(HipcubBlockRadixSort, SortKeys) } else { - keys_output = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + keys_output + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value); } // Calculate expected results on host @@ -570,11 +570,11 @@ TYPED_TEST(HipcubBlockRadixSort, SortKeysValues) } else { - keys_output = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + keys_output + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value); } std::vector values_output; @@ -588,11 +588,11 @@ TYPED_TEST(HipcubBlockRadixSort, SortKeysValues) } else { - values_output = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + values_output + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } using key_value = std::pair; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp index 11e078842c10..ee1be7338dfc 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ // hipcub API #include +#include // Params for tests template; - __shared__ - typename breduce_t::TempStorage temp_storage; - value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}); + __shared__ typename breduce_t::TempStorage temp_storage; + value = breduce_t(temp_storage).Reduce(value, hipcub::Sum()); if(hipThreadIdx_x == 0) { device_output_reductions[hipBlockIdx_x] = value; @@ -337,9 +337,8 @@ void reduce_valid_kernel(T* device_output, const unsigned int index = (hipBlockIdx_x * BlockSize) + hipThreadIdx_x; T value = device_output[index]; using breduce_t = hipcub::BlockReduce; - __shared__ - typename breduce_t::TempStorage temp_storage; - value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}, valid_items); + __shared__ typename breduce_t::TempStorage temp_storage; + value = breduce_t(temp_storage).Reduce(value, hipcub::Sum(), valid_items); if(hipThreadIdx_x == 0) { device_output_reductions[hipBlockIdx_x] = value; @@ -607,7 +606,7 @@ void reduce_array_kernel(T* device_output, T* device_output_reductions) T reduction; using breduce_t = hipcub::BlockReduce; __shared__ typename breduce_t::TempStorage temp_storage; - reduction = breduce_t(temp_storage).Reduce(in_out, test_utils::plus{}); + reduction = breduce_t(temp_storage).Reduce(in_out, hipcub::Sum()); if(hipThreadIdx_x == 0) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp index b969423be9f0..02250d546230 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -112,11 +112,11 @@ void block_run_length_decode_kernel(const ItemT* d_run_items, ItemT decoded_items[DecodedItemsPerThread]; block_run_length_decode.RunLengthDecode(decoded_items, decoded_window_offset); - hipcub::StoreDirectBlocked(global_thread_idx, - d_decoded_items + decoded_window_offset, - decoded_items, - test_utils::minimum{}(total_decoded_size - decoded_window_offset, - decoded_items_per_block)); + hipcub::StoreDirectBlocked( + global_thread_idx, + d_decoded_items + decoded_window_offset, + decoded_items, + hipcub::Min{}(total_decoded_size - decoded_window_offset, decoded_items_per_block)); decoded_window_offset += decoded_items_per_block; } @@ -141,15 +141,13 @@ TYPED_TEST(HipcubBlockRunLengthDecodeTest, TestDecode) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); const LengthT max_run_length = static_cast( - _HIPCUB_STD::min(1000ll, - static_cast(_HIPCUB_STD::numeric_limits::max()))); + std::min(1000ll, static_cast(std::numeric_limits::max()))); size_t num_runs = runs_per_thread * block_size; - auto run_items - = test_utils::get_random_data(num_runs, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + auto run_items = test_utils::get_random_data(num_runs, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value); auto run_lengths = test_utils::get_random_data(num_runs, static_cast(1), max_run_length, @@ -162,8 +160,8 @@ TYPED_TEST(HipcubBlockRunLengthDecodeTest, TestDecode) const auto empty_run_items = test_utils::get_random_data(num_trailing_empty_runs, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), + std::numeric_limits::min(), + std::numeric_limits::max(), seed_value); // Not strictly required, but fixes a spurious GCC warning and good practice anyways run_items.reserve(run_items.size() + empty_run_items.size()); diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp index aef679bb5c42..edc0728aa05e 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -96,7 +96,7 @@ void inclusive_scan_kernel(T* device_output) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(value, value, test_utils::plus{}); + bscan_t(temp_storage).InclusiveScan(value, value, hipcub::Sum()); device_output[index] = value; } @@ -201,7 +201,7 @@ void inclusive_scan_initial_value_kernel(T* device_output, T initial_value) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(input, output, initial_value, test_utils::plus{}); + bscan_t(temp_storage).InclusiveScan(input, output, initial_value, hipcub::Sum()); for(unsigned int i = 0; i < ItemsPerThread; ++i) { @@ -304,7 +304,7 @@ void inclusive_scan_reduce_kernel(T* device_output, T* device_output_reductions) T reduction; using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(value, value, test_utils::plus{}, reduction); + bscan_t(temp_storage).InclusiveScan(value, value, hipcub::Sum(), reduction); device_output[index] = value; if(hipThreadIdx_x == 0) { @@ -438,7 +438,7 @@ void inclusive_scan_reduce_initial_value_kernel(T* device_output, using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(input, output, initial_value, test_utils::plus{}, reduction); + bscan_t(temp_storage).InclusiveScan(input, output, initial_value, hipcub::Sum(), reduction); for(unsigned int i = 0; i < ItemsPerThread; ++i) { @@ -579,7 +579,7 @@ void inclusive_scan_prefix_callback_kernel(T* device_output, T* device_output_bp using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(value, value, test_utils::plus{}, prefix_callback); + bscan_t(temp_storage).InclusiveScan(value, value, hipcub::Sum(), prefix_callback); device_output[index] = value; if(hipThreadIdx_x == 0) @@ -698,7 +698,7 @@ void exclusive_scan_kernel(T* device_output, T init) T value = device_output[index]; using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(value, value, init, test_utils::plus{}); + bscan_t(temp_storage).ExclusiveScan(value, value, init, hipcub::Sum()); device_output[index] = value; } @@ -795,7 +795,7 @@ void exclusive_scan_reduce_kernel(T* device_output, T* device_output_reductions, T reduction; using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(value, value, init, test_utils::plus{}, reduction); + bscan_t(temp_storage).ExclusiveScan(value, value, init, hipcub::Sum(), reduction); device_output[index] = value; if(hipThreadIdx_x == 0) { @@ -932,7 +932,7 @@ void exclusive_scan_prefix_callback_kernel(T* device_output, T* device_output_bp using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(value, value, test_utils::plus{}, prefix_callback); + bscan_t(temp_storage).ExclusiveScan(value, value, hipcub::Sum(), prefix_callback); device_output[index] = value; if(hipThreadIdx_x == 0) @@ -1907,7 +1907,7 @@ void inclusive_scan_array_kernel(T* device_output) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(in_out, in_out, test_utils::plus{}); + bscan_t(temp_storage).InclusiveScan(in_out, in_out, hipcub::Sum()); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2020,7 +2020,7 @@ void inclusive_scan_reduce_array_kernel(T* device_output, T* device_output_reduc using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; T reduction; - bscan_t(temp_storage).InclusiveScan(in_out, in_out, test_utils::plus{}, reduction); + bscan_t(temp_storage).InclusiveScan(in_out, in_out, hipcub::Sum(), reduction); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2171,7 +2171,7 @@ void inclusive_scan_array_prefix_callback_kernel(T* device_output, using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(in_out, in_out, test_utils::plus{}, prefix_callback); + bscan_t(temp_storage).InclusiveScan(in_out, in_out, hipcub::Sum(), prefix_callback); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2319,7 +2319,7 @@ void exclusive_scan_array_kernel(T* device_output, T init) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, test_utils::plus{}); + bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, hipcub::Sum()); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2440,7 +2440,7 @@ void exclusive_scan_reduce_array_kernel(T* device_output, T* device_output_reduc using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; T reduction; - bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, test_utils::plus{}, reduction); + bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, hipcub::Sum(), reduction); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2608,7 +2608,7 @@ void exclusive_scan_prefix_callback_array_kernel(T* device_output, using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(in_out, in_out, test_utils::plus{}, prefix_callback); + bscan_t(temp_storage).ExclusiveScan(in_out, in_out, hipcub::Sum(), prefix_callback); // store for(unsigned int j = 0; j < ItemsPerThread; j++) diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp index b81581552a9f..ea5b4376251b 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp @@ -99,8 +99,8 @@ TYPED_TEST(HipcubBlockShuffleTests, BlockOffset) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - int distance = rand() % _HIPCUB_STD::min(size_t(10), block_size / 2) - - _HIPCUB_STD::min(size_t(10), block_size / 2); + int distance + = rand() % std::min(size_t(10), block_size / 2) - std::min(size_t(10), block_size / 2); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value << " & distance = " << distance); // Generate data @@ -183,7 +183,7 @@ TYPED_TEST(HipcubBlockShuffleTests, BlockRotate) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - int distance = rand() % _HIPCUB_STD::min(size_t(5), block_size / 2); + int distance = rand() % std::min(size_t(5), block_size / 2); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value << " & distance = " << distance); // Generate data diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp index cb831ea02d3b..155b748d041e 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,9 @@ // hipcub API #include +#include +#include +#include #include "test_utils.hpp" #include "test_utils_data_generation.hpp" @@ -154,7 +157,7 @@ TYPED_TEST(HipcubDeviceAdjacentDifference, SubtractLeftCopy) static constexpr std::integral_constant copy_constant{}; using output_type = std::conditional_t; - static constexpr test_utils::minus op; + static constexpr ::hipcub::Difference op; hipStream_t stream = 0; if(TestFixture::params::use_graphs) @@ -457,7 +460,7 @@ TYPED_TEST(HipcubDeviceAdjacentDifferenceLargeTests, LargeIndicesAndOpOnce) static constexpr hipStream_t stream = 0; // default - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(std::size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -478,15 +481,15 @@ TYPED_TEST(HipcubDeviceAdjacentDifferenceLargeTests, LargeIndicesAndOpOnce) HIP_CHECK(hipMemset(d_counter, 0, sizeof(*d_counter))); OutputIterator output(d_incorrect_flag, d_counter); - const auto input = test_utils::counting_iterator(T{0}); + const auto input = rocprim::counting_iterator(T{0}); static constexpr auto left_tag = std::integral_constant{}; static constexpr auto copy_tag = std::integral_constant{}; FocusIndex op; // Allocate temporary storage - size_t temp_storage_size = 0; - void* d_temp_storage = nullptr; + std::size_t temp_storage_size = 0; + void* d_temp_storage = nullptr; HIP_CHECK(dispatch_adjacent_difference(left_tag, copy_tag, d_temp_storage, diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp index f9a3cf1d4f24..e446d77af0b5 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,6 +27,7 @@ #include "test_utils_types.hpp" #include +#include #include #include @@ -267,12 +268,12 @@ TYPED_TEST(DeviceBatchCopyTests, SizeAndTypeVariation) h_buffer_num_elements.end(), 0, src_offsets.begin(), - test_utils::plus{}); + hipcub::Sum{}); test_utils::host_exclusive_scan(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), 0, dst_offsets.begin(), - test_utils::plus{}); + hipcub::Sum{}); } // Generate the source and destination pointers. diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp index 5e291994a1f3..94fa2f8e59a0 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ // required hipcub headers #include +#include #include #include @@ -691,7 +692,7 @@ TEST(HipcubDeviceForTests, ForCountingIterator) // Device pointers unsigned int* d_count; - const auto it = test_utils::counting_iterator{0}; + const auto it = rocprim::counting_iterator{0}; // Allocate memory HIP_CHECK(test_common_utils::hipMallocHelper(&d_count, sizeof(unsigned int))); @@ -744,7 +745,7 @@ TEST(HipcubDeviceForTests, ForCopyCountingIterator) // Device pointers unsigned int* d_count; - const auto it = test_utils::counting_iterator{0}; + const auto it = rocprim::counting_iterator{0}; // Allocate memory HIP_CHECK(test_common_utils::hipMallocHelper(&d_count, sizeof(unsigned int))); @@ -853,7 +854,7 @@ TEST(HipcubDeviceForTests, ForEachCopyNTempStore) } // ForEachInExtents only enables when the cccl mdspan extension is enabled -#if defined(__HIP_PLATFORM_NVIDIA__) || defined(__HIP_PLATFORM_AMD__) +#if(defined(__HIP_PLATFORM_NVIDIA__) && defined(__cccl_lib_mdspan)) || defined(__HIP_PLATFORM_AMD__) template struct HipcubTestParamsMerge @@ -896,19 +897,19 @@ struct HipcubDeviceForEachInExtentsTests : public ::testing::Test template using HipcubDeviceForEachInExtentsParamGenerator - = ::testing::Types>, - DeviceForEachInExtentsParams<::test_utils::extents>, - DeviceForEachInExtentsParams<::test_utils::extents>, - DeviceForEachInExtentsParams<::test_utils::extents>, - DeviceForEachInExtentsParams<::test_utils::extents>>; + = ::testing::Types>, + DeviceForEachInExtentsParams<::hipcub::extents>, + DeviceForEachInExtentsParams<::hipcub::extents>, + DeviceForEachInExtentsParams<::hipcub::extents>, + DeviceForEachInExtentsParams<::hipcub::extents>>; using HipcubDeviceForEachInExtentsTestsParams = typename HipcubTestParamsMergeAll< HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator<_HIPCUB_STD::int64_t>, - HipcubDeviceForEachInExtentsParamGenerator>::type; + HipcubDeviceForEachInExtentsParamGenerator, + HipcubDeviceForEachInExtentsParamGenerator, + HipcubDeviceForEachInExtentsParamGenerator, + HipcubDeviceForEachInExtentsParamGenerator, + HipcubDeviceForEachInExtentsParamGenerator>::type; template -inline void fill_linear(std::vector& vector, - const ::test_utils::extents& ext) +inline void fill_linear(std::vector& vector, const ::hipcub::extents& ext) { size_t pos = 0; fill_linear_impl(vector, ext, pos); @@ -964,22 +964,6 @@ struct LinearStore } }; -template -struct ForEachInExtentsOp - -{ - using op_data_t = item_t[3]; - void* d_data; - - __device__ __host__ __forceinline__ - void operator()(int idx, int x, int y, int z) - { - auto& i = static_cast(d_data)[idx]; - // We use the "placement new" operator to copy the data from an initializer list. - new(&i) op_data_t{x, y, z}; - } -}; - TYPED_TEST_SUITE(HipcubDeviceForEachInExtentsTests, HipcubDeviceForEachInExtentsTestsParams); TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsAPI) @@ -990,8 +974,8 @@ TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsAPI) using item_t = int; using data_t = std::array; - using extents_type = test_utils::extents; - constexpr auto extents_size = test_utils::extents_size::value; + using extents_type = hipcub::extents; + constexpr auto extents_size = hipcub::extents_size::value; constexpr auto memory_size = extents_size * sizeof(data_t); constexpr extents_type ext{}; @@ -1015,7 +999,21 @@ TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsAPI) HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, memory_size)); HIP_CHECK(hipMemset(d_input, 0, memory_size)); - HIP_CHECK(hipcub::DeviceFor::ForEachInExtents(ext, ForEachInExtentsOp{d_input})); + struct Op + { + using op_data_t = item_t[3]; + void* d_data; + + __device__ __host__ __forceinline__ + void operator()(int idx, int x, int y, int z) + { + auto& i = static_cast(d_data)[idx]; + // We use the "placement new" operator to copy the data from an initializer list. + new(&i) op_data_t{x, y, z}; + } + }; + + HIP_CHECK(hipcub::DeviceFor::ForEachInExtents(ext, Op{d_input})); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); @@ -1035,7 +1033,7 @@ TYPED_TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsStatic) using item_t = index_type; using data_t = std::array; - constexpr auto extents_size = test_utils::extents_size::value; + constexpr auto extents_size = hipcub::extents_size::value; constexpr auto memory_size = extents_size * sizeof(data_t); constexpr auto rank = extents_type::rank(); using store_op_t = LinearStore; @@ -1067,16 +1065,16 @@ TYPED_TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsStatic) HIP_CHECK(hipFree(d_input)); } -#endif // defined(__HIP_PLATFORM_NVIDIA__) || defined(__HIP_PLATFORM_AMD__) +#endif // (defined(__HIP_PLATFORM_NVIDIA__) && defined(__cccl_lib_mdspan)) || defined(__HIP_PLATFORM_AMD__) template class HipcubDeviceForBulkTests : public HipcubDeviceForTests {}; -using HipcubDeviceForBulkTestsParams = ::testing::Types, - DeviceForParams, - DeviceForParams<_HIPCUB_STD::int64_t>, - DeviceForParams>; +using HipcubDeviceForBulkTestsParams = ::testing::Types, + DeviceForParams, + DeviceForParams, + DeviceForParams>; TYPED_TEST_SUITE(HipcubDeviceForBulkTests, HipcubDeviceForBulkTestsParams); @@ -1089,7 +1087,7 @@ struct offset_count_device_t HIPCUB_DEVICE void operator()(OffsetT i) { - static_assert(std::is_same_v, "T and OffsetT must be the same type"); + static_assert(std::is_same::value, "T and OffsetT must be the same type"); atomicAdd(d_count + i, 1); } }; diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp index 48ed8092f9f3..1433f69e56ac 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -30,6 +30,8 @@ // hipcub API #include +#include +#include // rows, columns, (row_stride - columns * Channels) std::vector> get_dims() @@ -71,11 +73,9 @@ inline auto get_random_samples(size_t size, U min, U max, unsigned int seed_valu return test_utils::get_random_data( size, static_cast( - _HIPCUB_STD::max(min1 - d / 10, - static_cast(_HIPCUB_STD::numeric_limits::lowest()))), + std::max(min1 - d / 10, static_cast(std::numeric_limits::lowest()))), static_cast( - _HIPCUB_STD::min(max1 + d / 10, - static_cast(_HIPCUB_STD::numeric_limits::max()))), + std::min(max1 + d / 10, static_cast(std::numeric_limits::max()))), seed_value); } @@ -89,11 +89,8 @@ inline auto get_random_samples(size_t size, U min, U max, unsigned int seed_valu return test_utils::get_random_data( size, static_cast( - _HIPCUB_STD::max(min1 - d / 10, - static_cast(_HIPCUB_STD::numeric_limits::lowest()))), - static_cast( - _HIPCUB_STD::min(max1 + d / 10, - static_cast(_HIPCUB_STD::numeric_limits::max()))), + std::max(min1 - d / 10, static_cast(std::numeric_limits::lowest()))), + static_cast(std::min(max1 + d / 10, static_cast(std::numeric_limits::max()))), seed_value); } @@ -198,7 +195,7 @@ TYPED_TEST(HipcubDeviceHistogramEven, Even) const size_t row_stride = columns + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = _HIPCUB_STD::max(1, rows * row_stride); + const size_t size = std::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -374,7 +371,7 @@ TYPED_TEST(HipcubDeviceHistogramEvenOverflow, EvenOverflow) const native_level_type n_lower_level = 0; const native_level_type n_upper_level - = static_cast(_HIPCUB_STD::numeric_limits::max()); + = static_cast(std::numeric_limits::max()); level_type lower_level = test_utils::convert_to_device(n_lower_level); level_type upper_level = test_utils::convert_to_device(n_upper_level); @@ -390,7 +387,7 @@ TYPED_TEST(HipcubDeviceHistogramEvenOverflow, EvenOverflow) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); // Generate data - auto d_input = test_utils::counting_iterator{0UL}; + auto d_input = rocprim::counting_iterator{0UL}; counter_type* d_histogram; HIP_CHECK(test_common_utils::hipMallocHelper(&d_histogram, bins * sizeof(counter_type))); @@ -473,9 +470,7 @@ using Params2 = ::testing::Types< params2, params2, params2, -#if defined(__HIP_PLATFORM_AMD__) params2, -#endif params2, params2, params2, @@ -527,7 +522,7 @@ TYPED_TEST(HipcubDeviceHistogramRange, Range) const size_t row_stride = columns + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = _HIPCUB_STD::max(1, rows * row_stride); + const size_t size = std::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -773,14 +768,6 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) upper_level[channel] = test_utils::convert_to_device(n_upper_level[channel]); } - // accuracy problems with bfloat and half - // nvidia cub also doesn't work - // TODO: check if nvidia works with only sample type bfloat/half - if(test_utils::is_half::value || test_utils::is_bfloat16::value) - { - GTEST_SKIP(); - } - hipStream_t stream = 0; // default if(TestFixture::params::use_graphs) { @@ -800,7 +787,7 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) const size_t row_stride = columns * channels + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = _HIPCUB_STD::max(1, rows * row_stride); + const size_t size = std::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -808,9 +795,9 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); std::vector channel_seeds = test_utils::get_random_data( - _HIPCUB_STD::max(size, static_cast(channels)), - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), + std::max(size, static_cast(channels)), + std::numeric_limits::min(), + std::numeric_limits::max(), seed_value + seed_value_addition // Make sure that we do not use the same or shifted sequence ); @@ -893,9 +880,8 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) } } } - test_utils::transform_iterator> d_input2( - d_input, - transform_op()); + rocprim::transform_iterator, sample_type> + d_input2(d_input, transform_op()); size_t temporary_storage_bytes = 0; if(rows == 1) { @@ -1040,9 +1026,7 @@ using Params4 = ::testing::Types< params4, params4, params4, -#if defined(__HIP_PLATFORM_AMD__) params4, -#endif params4, params4, params4, @@ -1105,7 +1089,7 @@ TYPED_TEST(HipcubDeviceHistogramMultiRange, MultiRange) const size_t row_stride = columns * channels + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = _HIPCUB_STD::max(1, rows * row_stride); + const size_t size = std::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -1113,9 +1097,9 @@ TYPED_TEST(HipcubDeviceHistogramMultiRange, MultiRange) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); std::vector channel_seeds = test_utils::get_random_data( - _HIPCUB_STD::max(size, static_cast(channels)), - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), + std::max(size, static_cast(channels)), + std::numeric_limits::min(), + std::numeric_limits::max(), seed_value); // Generate data @@ -1226,9 +1210,8 @@ TYPED_TEST(HipcubDeviceHistogramMultiRange, MultiRange) } } } - test_utils::transform_iterator> d_input2( - d_input, - transform_op()); + rocprim::transform_iterator, sample_type> + d_input2(d_input, transform_op()); size_t temporary_storage_bytes = 0; if(rows == 1) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp index 342214e995c6..b2b997ff8f10 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,6 +27,7 @@ #include "test_utils_types.hpp" #include +#include #include #include @@ -271,12 +272,12 @@ TYPED_TEST(DeviceBatchMemcpyTests, SizeAndTypeVariation) h_buffer_num_elements.end(), 0, src_offsets.begin(), - test_utils::plus{}); + hipcub::Sum{}); test_utils::host_exclusive_scan(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), 0, dst_offsets.begin(), - test_utils::plus{}); + hipcub::Sum{}); } // Generate the source and destination pointers. @@ -337,306 +338,3 @@ TYPED_TEST(DeviceBatchMemcpyTests, SizeAndTypeVariation) HIP_CHECK(hipFree(d_temp_storage)); } - -TEST(DeviceMemcpyBatched, ZeroBuffersNoOp) -{ - using T = uint8_t; - T** d_srcs = nullptr; - T** d_dsts = nullptr; - size_t* d_sizes = nullptr; - - size_t temp_bytes = 0; - HIP_CHECK(hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, 0)); - void* d_temp = nullptr; - if(temp_bytes) - { - HIP_CHECK(hipMalloc(&d_temp, temp_bytes)); - } - - // Should be a no-op without crashing - HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_temp, - temp_bytes, - d_srcs, - d_dsts, - d_sizes, - 0, - hipStreamDefault)); - if(d_temp) - { - HIP_CHECK(hipFree(d_temp)); - } -} - -TEST(DeviceMemcpyBatched, ZeroSizeEntries) -{ - using T = uint8_t; - const int num_buffers = 5; - const std::vector h_sizes = {0, 1, 0, 7, 0}; - - size_t total = 0; - for(auto s : h_sizes) - { - total += s; - } - - T* d_input = nullptr; - T* d_output = nullptr; - HIP_CHECK(hipMalloc(&d_input, total)); - HIP_CHECK(hipMalloc(&d_output, total)); - - // Build src/dst arrays - std::vector h_srcs(num_buffers), h_dsts(num_buffers); - size_t offset = 0; - for(int i = 0; i < num_buffers; ++i) - { - h_srcs[i] = d_input + offset; - h_dsts[i] = d_output + offset; - offset += h_sizes[i]; - } - - // Fill input - std::vector h_in(total); - std::iota(h_in.begin(), h_in.end(), static_cast(3)); - HIP_CHECK(hipMemcpy(d_input, h_in.data(), total, hipMemcpyHostToDevice)); - - // Device arrays - T** d_srcs = nullptr; - T** d_dsts = nullptr; - size_t* d_sizes = nullptr; - HIP_CHECK(hipMalloc(&d_srcs, num_buffers * sizeof(T*))); - HIP_CHECK(hipMalloc(&d_dsts, num_buffers * sizeof(T*))); - HIP_CHECK(hipMalloc(&d_sizes, num_buffers * sizeof(size_t))); - HIP_CHECK(hipMemcpy(d_srcs, h_srcs.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_dsts, h_dsts.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_sizes, h_sizes.data(), num_buffers * sizeof(size_t), hipMemcpyHostToDevice)); - - // Temp storage - size_t temp_bytes = 0; - HIP_CHECK( - hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); - void* d_temp = nullptr; - HIP_CHECK(hipMalloc(&d_temp, temp_bytes)); - - HIP_CHECK( - hipcub::DeviceMemcpy::Batched(d_temp, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); - - // Verify - std::vector h_out(total); - HIP_CHECK(hipMemcpy(h_out.data(), d_output, total, hipMemcpyDeviceToHost)); - EXPECT_EQ(h_in, h_out); - - HIP_CHECK(hipFree(d_temp)); - HIP_CHECK(hipFree(d_sizes)); - HIP_CHECK(hipFree(d_dsts)); - HIP_CHECK(hipFree(d_srcs)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_input)); -} - -TEST(DeviceMemcpyBatched, NonDefaultStreamAndTempReuse) -{ - using T = uint32_t; - const int num_buffers = 8; - std::mt19937 rng(123); - std::uniform_int_distribution dist(1, 4096); - std::vector sizes(num_buffers); - size_t total = 0; - for(int i = 0; i < num_buffers; ++i) - { - sizes[i] = dist(rng) * sizeof(T); - total += sizes[i]; - } - - T* d_in = nullptr; - T* d_out = nullptr; - HIP_CHECK(hipMalloc(&d_in, total)); - HIP_CHECK(hipMalloc(&d_out, total)); - - // Fill input - std::vector h_in(total / sizeof(T)); - for(size_t i = 0; i < h_in.size(); ++i) - h_in[i] = static_cast(i ^ 0xDEADBEEF); - HIP_CHECK(hipMemcpy(d_in, h_in.data(), total, hipMemcpyHostToDevice)); - - // Build offset/size pairs - struct Chunk - { - size_t offset; - size_t size; - }; - std::vector chunks(num_buffers); - size_t acc = 0; - for(int i = 0; i < num_buffers; ++i) - { - chunks[i].offset = acc; - chunks[i].size = sizes[i]; - acc += sizes[i]; - } - std::shuffle(chunks.begin(), chunks.end(), rng); - - std::vector h_srcs(num_buffers), h_dsts(num_buffers); - std::vector h_sizes(num_buffers); - for(int i = 0; i < num_buffers; ++i) - { - h_srcs[i] = d_in + chunks[i].offset / sizeof(T); - h_dsts[i] = d_out + chunks[i].offset / sizeof(T); - h_sizes[i] = chunks[i].size; - } - - T** d_srcs = nullptr; - T** d_dsts = nullptr; - size_t* d_sizes = nullptr; - HIP_CHECK(hipMalloc(&d_srcs, num_buffers * sizeof(T*))); - HIP_CHECK(hipMalloc(&d_dsts, num_buffers * sizeof(T*))); - HIP_CHECK(hipMalloc(&d_sizes, num_buffers * sizeof(size_t))); - - // Setup stream and event - hipStream_t setup; - hipEvent_t ready; - HIP_CHECK(hipStreamCreate(&setup)); - HIP_CHECK(hipEventCreateWithFlags(&ready, hipEventDisableTiming)); - - HIP_CHECK(hipMemcpyAsync(d_srcs, - h_srcs.data(), - num_buffers * sizeof(T*), - hipMemcpyHostToDevice, - setup)); - HIP_CHECK(hipMemcpyAsync(d_dsts, - h_dsts.data(), - num_buffers * sizeof(T*), - hipMemcpyHostToDevice, - setup)); - HIP_CHECK(hipMemcpyAsync(d_sizes, - h_sizes.data(), - num_buffers * sizeof(size_t), - hipMemcpyHostToDevice, - setup)); - HIP_CHECK(hipEventRecord(ready, setup)); - - // Query temp storage - size_t temp_bytes = 0; - HIP_CHECK( - hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); - - void* d_tempA = nullptr; - void* d_tempB = nullptr; - HIP_CHECK(hipMalloc(&d_tempA, temp_bytes)); - HIP_CHECK(hipMalloc(&d_tempB, temp_bytes)); - - hipStream_t streamA, streamB; - HIP_CHECK(hipStreamCreate(&streamA)); - HIP_CHECK(hipStreamCreate(&streamB)); - HIP_CHECK(hipStreamWaitEvent(streamA, ready, 0)); - HIP_CHECK(hipStreamWaitEvent(streamB, ready, 0)); - - // Launch batched memcpy - HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_tempA, - temp_bytes, - d_srcs, - d_dsts, - d_sizes, - num_buffers, - streamA)); - HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_tempB, - temp_bytes, - d_srcs, - d_dsts, - d_sizes, - num_buffers, - streamB)); - - HIP_CHECK(hipStreamSynchronize(streamA)); - HIP_CHECK(hipStreamSynchronize(streamB)); - - // Verify - std::vector h_out(h_in.size()); - HIP_CHECK(hipMemcpy(h_out.data(), d_out, total, hipMemcpyDeviceToHost)); - - for(size_t i = 0; i < h_in.size(); ++i) - { - EXPECT_EQ(h_in[i], h_out[i]) << "Mismatch at index " << i; - } - - // Cleanup - HIP_CHECK(hipEventDestroy(ready)); - HIP_CHECK(hipStreamDestroy(setup)); - HIP_CHECK(hipStreamDestroy(streamA)); - HIP_CHECK(hipStreamDestroy(streamB)); - HIP_CHECK(hipFree(d_tempA)); - HIP_CHECK(hipFree(d_tempB)); - HIP_CHECK(hipFree(d_sizes)); - HIP_CHECK(hipFree(d_dsts)); - HIP_CHECK(hipFree(d_srcs)); - HIP_CHECK(hipFree(d_out)); - HIP_CHECK(hipFree(d_in)); -} - -struct PackedPair -{ - uint16_t a; - uint16_t b; -}; // alignment-sensitive -TEST(DeviceMemcpyBatched, PackedStructAlignment) -{ - using T = PackedPair; - const int num_buffers = 4; - const size_t elems_per_buffer = 1024; - const size_t bytes_per_buffer = elems_per_buffer * sizeof(T); - const size_t total_bytes = num_buffers * bytes_per_buffer; - - T* d_in = nullptr; - T* d_out = nullptr; - HIP_CHECK(hipMalloc(&d_in, total_bytes)); - HIP_CHECK(hipMalloc(&d_out, total_bytes)); - - std::vector h_in(num_buffers * elems_per_buffer); - for(size_t i = 0; i < h_in.size(); ++i) - { - h_in[i] = T{static_cast(i), static_cast(~i)}; - } - HIP_CHECK(hipMemcpy(d_in, h_in.data(), total_bytes, hipMemcpyHostToDevice)); - - std::vector h_sizes(num_buffers, bytes_per_buffer); - std::vector h_srcs(num_buffers), h_dsts(num_buffers); - for(int i = 0; i < num_buffers; ++i) - { - h_srcs[i] = d_in + i * elems_per_buffer; - h_dsts[i] = d_out + i * elems_per_buffer; - } - - T** d_srcs = nullptr; - T** d_dsts = nullptr; - size_t* d_sizes = nullptr; - HIP_CHECK(hipMalloc(&d_srcs, num_buffers * sizeof(T*))); - HIP_CHECK(hipMalloc(&d_dsts, num_buffers * sizeof(T*))); - HIP_CHECK(hipMalloc(&d_sizes, num_buffers * sizeof(size_t))); - HIP_CHECK(hipMemcpy(d_srcs, h_srcs.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_dsts, h_dsts.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_sizes, h_sizes.data(), num_buffers * sizeof(size_t), hipMemcpyHostToDevice)); - - size_t temp_bytes = 0; - HIP_CHECK( - hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); - void* d_temp = nullptr; - HIP_CHECK(hipMalloc(&d_temp, temp_bytes)); - - HIP_CHECK( - hipcub::DeviceMemcpy::Batched(d_temp, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); - - std::vector h_out(h_in.size()); - HIP_CHECK(hipMemcpy(h_out.data(), d_out, total_bytes, hipMemcpyDeviceToHost)); - EXPECT_TRUE(std::equal(h_in.begin(), - h_in.end(), - h_out.begin(), - [](const PackedPair& x, const PackedPair& y) - { return x.a == y.a && x.b == y.b; })); - - HIP_CHECK(hipFree(d_temp)); - HIP_CHECK(hipFree(d_sizes)); - HIP_CHECK(hipFree(d_dsts)); - HIP_CHECK(hipFree(d_srcs)); - HIP_CHECK(hipFree(d_out)); - HIP_CHECK(hipFree(d_in)); -} diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp index d07aa79d3537..c04f35afa759 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ // hipcub API #include +#include #include "identity_iterator.hpp" #include "test_utils_data_generation.hpp" @@ -468,16 +469,11 @@ std::vector> get_large_sizes() TEST(HipcubDeviceMerge, MergeLargeSizeIterators) { - -#if defined(_WIN32) - GTEST_SKIP() << "Windows AMD HIP cannot allocate >= 4 GiB buffers."; -#endif - int device_id = test_common_utils::obtain_device_from_ctest(); SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using key_type = _HIPCUB_STD::int64_t; + using key_type = int; using compare_function = test_utils::less; hipStream_t stream = 0; // default @@ -500,9 +496,9 @@ TEST(HipcubDeviceMerge, MergeLargeSizeIterators) compare_function compare_op; // Generate data - const auto input1 = test_utils::counting_iterator(key_type{0}); + const auto input1 = rocprim::counting_iterator(key_type{0}); const auto input2 - = test_utils::counting_iterator(key_type{static_cast(size1)}); + = rocprim::counting_iterator(key_type{static_cast(size1)}); std::vector vec_input1(size1); std::vector vec_input2(size2); std::iota(vec_input1.begin(), vec_input1.end(), 0); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in index 2731de1cbc46..94daf870e630 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in +++ b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -50,7 +50,7 @@ #endif #if HIPCUB_TEST_TYPE_SLICE == 0 -#if _CCCL_HAS_INT128() +#if HIPCUB_IS_INT128_ENABLED INSTANTIATE(params<__uint128_t, __uint128_t, true >) INSTANTIATE(params<__int128_t, __int128_t, true >) #endif diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp index a8508b3087a5..9ce9c64e0f1e 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -82,8 +82,9 @@ class HipcubDeviceRadixSort : public ::testing::Test TYPED_TEST_SUITE_P(HipcubDeviceRadixSort); template -auto generate_key_input(size_t size, unsigned int seed_value) - -> std::enable_if_t<_HIPCUB_STD::is_floating_point_v, std::vector> +auto generate_key_input(size_t size, unsigned int seed_value) HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + -> std::enable_if_t::CATEGORY == hipcub::FLOATING_POINT, + std::vector> HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP { auto result = test_utils::get_random_data(size, test_utils::numeric_limits::min(), @@ -94,8 +95,9 @@ auto generate_key_input(size_t size, unsigned int seed_value) } template -auto generate_key_input(size_t size, unsigned int seed_value) - -> std::enable_if_t, std::vector> +auto generate_key_input(size_t size, unsigned int seed_value) HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + -> std::enable_if_t::CATEGORY != hipcub::FLOATING_POINT, + std::vector> HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP { using inner_t = typename test_utils::inner_type::type; return test_utils::get_random_data(size, @@ -1227,18 +1229,12 @@ inline void sort_keys_over_4g() SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using key_type = uint32_t; + using key_type = uint8_t; constexpr unsigned int start_bit = 0; constexpr unsigned int end_bit = 8ull * sizeof(key_type); constexpr hipStream_t stream = 0; - - constexpr size_t total_bytes = (1ull << 32) + 32; - static_assert(total_bytes > (1ull << 32), "must be over 4 GiB"); - static_assert(total_bytes % sizeof(key_type) == 0, - "total_bytes must be divisible by sizeof(key_type)"); - - constexpr size_t size = total_bytes / sizeof(key_type); - constexpr size_t number_of_possible_keys = 1ull << (8ull * sizeof(key_type)); + constexpr size_t size = (1ull << 32) + 32; + constexpr size_t number_of_possible_keys = 1ull << (8ull * sizeof(key_type)); assert(std::is_unsigned::value); hipDeviceProp_t dev_prop; HIP_CHECK(hipGetDeviceProperties(&dev_prop, device_id)); @@ -1269,8 +1265,8 @@ inline void sort_keys_over_4g() std::vector keys_input = test_utils::get_random_data(size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), + std::numeric_limits::min(), + std::numeric_limits::max(), seed_value); //generate histogram of the randomly generated values @@ -1324,7 +1320,7 @@ inline void sort_keys_over_4g() hipMemcpyDeviceToHost)); size_t counter = 0; - for(size_t i = 0; i <= _HIPCUB_STD::numeric_limits::max(); ++i) + for(size_t i = 0; i <= std::numeric_limits::max(); ++i) { for(size_t j = 0; j < histogram[i]; ++j) { @@ -1362,14 +1358,6 @@ inline void sort_keys_large_sizes() { SCOPED_TRACE(testing::Message() << "with size = " << size); - // Avoid sizes the CUB backend can't handle -#ifdef __HIP_PLATFORM_NVIDIA__ - if(size > static_cast(::cuda::std::numeric_limits::max())) - { - continue; - } -#endif // __HIP_PLATFORM_NVIDIA__ - // Generate data std::vector keys_input; try @@ -1384,10 +1372,6 @@ inline void sort_keys_large_sizes() key_type* d_keys; HIP_CHECK_MEMORY(test_common_utils::hipMallocHelper(&d_keys, size * sizeof(key_type))); - - key_type* d_keys_out; - HIP_CHECK_MEMORY(test_common_utils::hipMallocHelper(&d_keys_out, size * sizeof(key_type))); - HIP_CHECK( hipMemcpy(d_keys, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); @@ -1396,7 +1380,7 @@ inline void sort_keys_large_sizes() HIP_CHECK(invoke_sort_keys(d_temporary_storage, temporary_storage_bytes, d_keys, - d_keys_out, + d_keys, size, start_bit, end_bit, @@ -1410,7 +1394,7 @@ inline void sort_keys_large_sizes() HIP_CHECK(invoke_sort_keys(d_temporary_storage, temporary_storage_bytes, d_keys, - d_keys_out, + d_keys, size, start_bit, end_bit, @@ -1419,16 +1403,23 @@ inline void sort_keys_large_sizes() HIP_CHECK(hipFree(d_temporary_storage)); std::vector keys_output(size); - HIP_CHECK(hipMemcpy(keys_output.data(), - d_keys_out, - size * sizeof(key_type), - hipMemcpyDeviceToHost)); + try + { + keys_output.resize(size); + } + catch(const std::bad_alloc& e) + { + HIP_CHECK(hipFree(d_keys)); + continue; + } + + HIP_CHECK( + hipMemcpy(keys_output.data(), d_keys, size * sizeof(key_type), hipMemcpyDeviceToHost)); - HIP_CHECK(hipFree(d_keys_out)); HIP_CHECK(hipFree(d_keys)); // Check if output values are as expected - const size_t unique_keys = size_t(_HIPCUB_STD::numeric_limits::max()) + 1; + const size_t unique_keys = size_t(std::numeric_limits::max()) + 1; const size_t segment_length = test_utils::ceiling_div(size, unique_keys); const size_t full_segments = size % unique_keys == 0 ? unique_keys : size % unique_keys; for(size_t i = 0; i < size; i += 4321) diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp index 41e3b64d51b6..2866cdd76d0a 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -29,6 +29,7 @@ // hipcub API #include +#include // Params for tests template @@ -61,39 +62,17 @@ using HipcubDeviceReduceTestsParams = ::testing::Types< DeviceReduceParams, DeviceReduceParams, DeviceReduceParams, - DeviceReduceParams, - DeviceReduceParams, + DeviceReduceParams #ifdef __HIP_PLATFORM_AMD__ - DeviceReduceParams, -#endif + , + DeviceReduceParams, // Doesn't work on NVIDIA / CUB + DeviceReduceParams, // Doesn't work on NVIDIA / CUB DeviceReduceParams, test_utils::custom_test_type>, - DeviceReduceParams, test_utils::custom_test_type>>; - -// Device numeric_limits for custom_test_type -_HIPCUB_STD_NAMESPACE_BEGIN - -template -struct numeric_limits> -{ - static constexpr bool is_specialized = true; - - static constexpr test_utils::custom_test_type min() noexcept - { - return {_HIPCUB_STD::numeric_limits::min(), _HIPCUB_STD::numeric_limits::min()}; - } - - static constexpr test_utils::custom_test_type lowest() noexcept - { - return {_HIPCUB_STD::numeric_limits::lowest(), _HIPCUB_STD::numeric_limits::lowest()}; - } - - static constexpr test_utils::custom_test_type max() noexcept - { - return {_HIPCUB_STD::numeric_limits::max(), _HIPCUB_STD::numeric_limits::max()}; - } -}; - -_HIPCUB_STD_NAMESPACE_END + DeviceReduceParams, test_utils::custom_test_type> +#endif + >; TYPED_TEST_SUITE(HipcubDeviceReduceTests, HipcubDeviceReduceTestsParams); @@ -144,12 +123,11 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceSum) HIP_CHECK(hipDeviceSynchronize()); // Calculate expected results on host using the same accumulator type than on device - using Sum = typename AlgebraicSelector:: - type; // For custom_type_test tests + using Sum = + typename AlgebraicSelector::type; // For custom_type_test tests using AccumT = hipcub::detail::accumulator_t; Sum sum_op; - AccumT tmp_result - = AccumT(0.0f); // test_utils::plus uses as initial type the output type + AccumT tmp_result = AccumT(0.0f); // hipcub::Sum uses as initial type the output type for(unsigned int i = 0; i < input.size(); i++) { tmp_result = sum_op(tmp_result, input[i]); @@ -160,15 +138,15 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceSum) size_t temp_storage_size_bytes; void* d_temp_storage = nullptr; // Get size of d_temp_storage - if constexpr(std::is_same_v - || std::is_same_v) + if constexpr(std::is_same::value + || std::is_same::value) { HIP_CHECK(hipcub::DeviceReduce::Reduce(d_temp_storage, temp_storage_size_bytes, d_input, d_output, input.size(), - ExtendedFloatBinOp(), + ExtendedFloatBinOp(), U(0.f), stream)); } @@ -194,15 +172,15 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceSum) gHelper.startStreamCapture(stream); // Run - if constexpr(std::is_same_v - || std::is_same_v) + if constexpr(std::is_same::value + || std::is_same::value) { HIP_CHECK(hipcub::DeviceReduce::Reduce(d_temp_storage, temp_storage_size_bytes, d_input, d_output, input.size(), - ExtendedFloatBinOp(), + ExtendedFloatBinOp(), U(0.f), stream)); } @@ -288,10 +266,10 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceMinimum) // Calculate expected results on host using the same accumulator type than on device using Min = typename MinSelector::type; // For custom_type_test tests - using AccumT = hipcub::detail::accumulator_t; + using AccumT = hipcub::detail::accumulator_t; Min min_op; AccumT tmp_result = test_utils::numeric_limits< - AccumT>::max(); // test_utils::minimum uses as initial type the input type + AccumT>::max(); // hipcub::Min uses as initial type the input type for(unsigned int i = 0; i < input.size(); i++) { tmp_result = min_op(tmp_result, input[i]); @@ -402,7 +380,7 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceMaximum) // Calculate expected results on host using the same accumulator type than on device using Max = typename MaxSelector::type; // For custom_type_test tests - using AccumT = hipcub::detail::accumulator_t; + using AccumT = hipcub::detail::accumulator_t; Max max_op; AccumT tmp_result = test_utils::numeric_limits::min(); for(unsigned int i = 0; i < input.size(); i++) @@ -708,8 +686,8 @@ void test_argminmax2(typename TestFixture::input_type empty_value) using T = typename TestFixture::input_type; using Iterator = typename hipcub::ArgIndexInputIterator; using argidx_type = typename Iterator::value_type; - using extremum_type = decltype(std::declval().value); - using index_type = decltype(std::declval().key); + using extremum_type = typename argidx_type::value_type; + using index_type = typename argidx_type::key_type; DispatchFunction function; @@ -908,15 +886,13 @@ void test_argminmax_allinf(TypeParam value, TypeParam empty_value) if(size > 0) { // all +/- infinity should produce +/- infinity - ASSERT_NO_FATAL_FAILURE( - test_utils::assert_eq(output[0].key, static_cast(0))); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 0)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, value)); } else { // empty input should produce a special value - ASSERT_NO_FATAL_FAILURE( - test_utils::assert_eq(output[0].key, static_cast(1))); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 1)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, empty_value)); } } @@ -976,15 +952,13 @@ void test_argminmax_extremum(TypeParam value, TypeParam empty_value) if(size > 0) { // all +/- infinity should produce +/- infinity - ASSERT_NO_FATAL_FAILURE( - test_utils::assert_eq(output[0].key, static_cast(0))); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 0)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, value)); } else { // empty input should produce a special value - ASSERT_NO_FATAL_FAILURE( - test_utils::assert_eq(output[0].key, static_cast(1))); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 1)); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, empty_value)); } } @@ -1076,14 +1050,14 @@ TYPED_TEST(HipcubDeviceReduceTests, TransformReduce) hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); // Calculate expected results on host using the same accumulator type than on device - using Sum = typename AlgebraicSelector:: - type; // For custom_type_test tests + using Sum = + typename AlgebraicSelector::type; // For custom_type_test tests using AccumT = hipcub::detail::accumulator_t; Sum reduction_op; TestTransformOp transform_op; const U init(10); - AccumT tmp_result = init; // test_utils::plus uses as initial type the output type + AccumT tmp_result = init; // hipcub::Sum uses as initial type the output type for(size_t i = 0; i < input.size(); ++i) { tmp_result = reduction_op(tmp_result, transform_op(input[i])); @@ -1186,7 +1160,7 @@ TYPED_TEST(HipcubDeviceReduceLargeIndicesTests, LargeIndices) using T = typename TestFixture::input_type; using U = typename TestFixture::output_type; - using IteratorType = test_utils::constant_iterator; + using IteratorType = rocprim::constant_iterator; const std::vector exponents = {30, 31, 32, 33, 34}; for(auto exponent : exponents) { @@ -1245,7 +1219,7 @@ TYPED_TEST(HipcubDeviceReduceLargeIndicesTests, LargeIndices) HIP_CHECK(hipDeviceSynchronize()); // Check if output values are as expected - const size_t result = output[0]; + const std::size_t result = output[0]; ASSERT_EQ(result, size); HIP_CHECK(hipFree(d_output)); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp index 901d8f034b7d..33f86c435600 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -52,23 +52,23 @@ class HipcubDeviceReduceByKey : public ::testing::Test { }; using Params = ::testing::Types< - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, // Sum for half and bfloat will result in values too big due to limited range. - params, - params, - params>; + params, + params, + params>; TYPED_TEST_SUITE(HipcubDeviceReduceByKey, Params); @@ -89,7 +89,7 @@ TYPED_TEST(HipcubDeviceReduceByKey, ReduceByKey) std::uniform_int_distribution>>::type; reduce_op_type reduce_op; - test_utils::equal key_compare_op; + hipcub::Equality key_compare_op; hipStream_t stream = 0; // default if(TestFixture::params::use_graphs) @@ -136,7 +136,7 @@ TYPED_TEST(HipcubDeviceReduceByKey, ReduceByKey) const size_t key_count = key_count_dis(gen); current_key += key_delta_dis(gen); - const size_t end = _HIPCUB_STD::min(size, offset + key_count); + const size_t end = std::min(size, offset + key_count); for(size_t i = offset; i < end; i++) { keys_input[i] = test_utils::convert_to_device(current_key); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp index 316a00943a16..797676b286c4 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -123,7 +123,7 @@ TYPED_TEST(HipcubDeviceRunLengthEncode, Encode) size_t key_count = key_count_dis(gen); current_key += key_delta_dis(gen); - const size_t end = _HIPCUB_STD::min(size, offset + key_count); + const size_t end = std::min(size, offset + key_count); key_count = end - offset; for(size_t i = offset; i < end; i++) { @@ -296,7 +296,7 @@ TYPED_TEST(HipcubDeviceRunLengthEncode, NonTrivialRuns) } current_key += key_delta_dis(gen); - const size_t end = _HIPCUB_STD::min(size, offset + key_count); + const size_t end = std::min(size, offset + key_count); key_count = end - offset; for(size_t i = offset; i < end; i++) { @@ -321,12 +321,12 @@ TYPED_TEST(HipcubDeviceRunLengthEncode, NonTrivialRuns) offset_type* d_offsets_output; count_type* d_counts_output; count_type* d_runs_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper( - &d_offsets_output, - _HIPCUB_STD::max(1, runs_count_expected) * sizeof(offset_type))); - HIP_CHECK(test_common_utils::hipMallocHelper( - &d_counts_output, - _HIPCUB_STD::max(1, runs_count_expected) * sizeof(count_type))); + HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets_output, + std::max(1, runs_count_expected) + * sizeof(offset_type))); + HIP_CHECK(test_common_utils::hipMallocHelper(&d_counts_output, + std::max(1, runs_count_expected) + * sizeof(count_type))); HIP_CHECK(test_common_utils::hipMallocHelper(&d_runs_count_output, sizeof(count_type))); size_t temporary_storage_bytes = 0; diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp index 92883f15c023..ee585fd590db 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,10 @@ // hipcub API #include +#include +#include +#include +#include #include "single_index_iterator.hpp" #include "test_utils_bfloat16.hpp" @@ -32,7 +36,7 @@ // Params for tests template struct DeviceScanParams @@ -61,22 +65,23 @@ class HipcubDeviceScanTests : public ::testing::Test static constexpr bool use_graphs = Params::use_graphs; }; -using HipcubDeviceScanTestsParams = ::testing::Types< - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams>; +using HipcubDeviceScanTestsParams + = ::testing::Types, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams>; // use float for accumulation of bfloat16 and half inputs if operator is plus template struct accum_type { - static constexpr bool is_low_precision = std::is_same_v - || std::is_same_v; + static constexpr bool is_low_precision + = std::is_same::value + || std::is_same::value; static constexpr bool is_add = test_utils::is_add_operator::value; using type = typename std::conditional_t; }; @@ -93,15 +98,14 @@ std::vector std::default_random_engine prng(seed_value); std::uniform_int_distribution segment_length_distribution(max_segment_length); - std::uniform_int_distribution key_distribution(_HIPCUB_STD::numeric_limits::max()); + std::uniform_int_distribution key_distribution(std::numeric_limits::max()); std::vector keys(size); size_t keys_start_index = 0; while(keys_start_index < size) { const size_t new_segment_length = segment_length_distribution(prng); - const size_t new_segment_end - = _HIPCUB_STD::min(size, keys_start_index + new_segment_length); + const size_t new_segment_end = std::min(size, keys_start_index + new_segment_length); const T key = key_distribution(prng); std::fill(std::next(keys.begin(), keys_start_index), std::next(keys.begin(), new_segment_end), @@ -117,7 +121,7 @@ TYPED_TEST(HipcubDeviceScanTests, AccumulatorTypeTest) using T = hipcub::detail::accumulator_t; using U = typename TestFixture::input_type; - static_assert(std::is_same_v, "accumulator type mismatch"); + static_assert(std::is_same::value, "accumulator type mismatch"); ASSERT_TRUE(true); } @@ -135,8 +139,8 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = test_utils::transform_iterator>; - constexpr bool inplace = std::is_same_v && std::is_same_v; + using IteratorType = rocprim::transform_iterator, acc_type>; + constexpr bool inplace = std::is_same::value && std::is_same::value; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then @@ -184,7 +188,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) T* d_input; U* d_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if constexpr(!inplace) + if HIPCUB_IF_CONSTEXPR(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); } @@ -207,9 +211,9 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if constexpr(std::is_same_v) + if HIPCUB_IF_CONSTEXPR(std::is_same::value) { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_size_bytes, @@ -229,7 +233,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) } else { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_size_bytes, @@ -277,7 +281,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -327,8 +331,8 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = test_utils::transform_iterator>; - constexpr bool inplace = std::is_same_v && std::is_same_v; + using IteratorType = rocprim::transform_iterator, acc_type>; + constexpr bool inplace = std::is_same::value && std::is_same::value; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then @@ -381,7 +385,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) T* d_input; U* d_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if constexpr(!inplace) + if HIPCUB_IF_CONSTEXPR(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); } @@ -408,7 +412,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceScan::InclusiveScanInit(d_temp_storage, temp_storage_size_bytes, @@ -458,7 +462,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -511,7 +515,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = test_utils::transform_iterator>; + using IteratorType = rocprim::transform_iterator, acc_type>; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then // as all conversions in the tests are to more precise types, @@ -579,7 +583,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) keys.begin(), expected.begin(), scan_op, - test_utils::equal()); + hipcub::Equality()); // Scan operator: CastOp. hipcub::CastOp op{}; @@ -591,7 +595,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) size_t temp_storage_size_bytes{}; void* d_temp_storage = nullptr; // Get size of d_temp_storage - if(std::is_same_v) + if(std::is_same::value) { HIP_CHECK(hipcub::DeviceScan::InclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -599,7 +603,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } else @@ -611,7 +615,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) d_output, scan_op, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } @@ -627,7 +631,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) gHelper.startStreamCapture(stream); // Run - if(std::is_same_v) + if(std::is_same::value) { HIP_CHECK(hipcub::DeviceScan::InclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -635,7 +639,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } else @@ -647,7 +651,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) d_output, scan_op, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } @@ -696,8 +700,8 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = test_utils::transform_iterator>; - constexpr bool inplace = std::is_same_v && std::is_same_v; + using IteratorType = rocprim::transform_iterator, acc_type>; + constexpr bool inplace = std::is_same::value && std::is_same::value; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then @@ -745,7 +749,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) T* d_input; U* d_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if constexpr(!inplace) + if HIPCUB_IF_CONSTEXPR(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); } @@ -759,7 +763,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) // Calculate expected results on host std::vector expected(input.size()); const T initial_value - = std::is_same_v + = std::is_same::value ? test_utils::convert_to_device(0) : test_utils::get_random_value(test_utils::convert_to_device(1), test_utils::convert_to_device(100), @@ -778,9 +782,9 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if constexpr(std::is_same_v) + if HIPCUB_IF_CONSTEXPR(std::is_same::value) { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_size_bytes, @@ -800,7 +804,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) } else { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_size_bytes, @@ -850,7 +854,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -901,7 +905,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = test_utils::transform_iterator>; + using IteratorType = rocprim::transform_iterator, acc_type>; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then // as all conversions in the tests are to more precise types, @@ -953,7 +957,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) test_utils::convert_to_device(10), seed_value); T initial_value = initial_value_vector.front(); - if(std::is_same_v) + if(std::is_same::value) { initial_value = test_utils::convert_to_device(0); } @@ -981,7 +985,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) initial_value, expected.begin(), scan_op, - test_utils::equal()); + hipcub::Equality()); // Scan operator: CastOp. hipcub::CastOp op{}; @@ -993,7 +997,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) size_t temp_storage_size_bytes; void* d_temp_storage = nullptr; // Get size of d_temp_storage - if(std::is_same_v) + if(std::is_same::value) { HIP_CHECK(hipcub::DeviceScan::ExclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -1001,7 +1005,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } else @@ -1014,7 +1018,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) scan_op, initial_value, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } @@ -1030,7 +1034,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) gHelper.startStreamCapture(stream); // Run - if(std::is_same_v) + if(std::is_same::value) { HIP_CHECK(hipcub::DeviceScan::ExclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -1038,7 +1042,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } else @@ -1051,7 +1055,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) scan_op, initial_value, static_cast(input.size()), - test_utils::equal(), + hipcub::Equality(), stream)); } @@ -1089,7 +1093,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) { using T = unsigned int; - using InputIterator = test_utils::counting_iterator; + using InputIterator = rocprim::counting_iterator; using OutputIterator = test_utils::single_index_iterator; const size_t size = (1ul << 31) + 1ul; @@ -1099,7 +1103,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) unsigned int seed_value = rand(); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); - // Create test_utils::counting_iterator with random starting point + // Create rocprim::counting_iterator with random starting point InputIterator input_begin(test_utils::get_random_value(0, 200, seed_value)); T* d_output; @@ -1116,7 +1120,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) temp_storage_size_bytes, input_begin, output_it, - test_utils::plus{}, + ::hipcub::Sum(), size, stream)); @@ -1132,7 +1136,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) temp_storage_size_bytes, input_begin, output_it, - test_utils::plus{}, + ::hipcub::Sum(), size, stream)); HIP_CHECK(hipGetLastError()); @@ -1159,7 +1163,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) { using T = unsigned int; - using InputIterator = test_utils::counting_iterator; + using InputIterator = rocprim::counting_iterator; using OutputIterator = test_utils::single_index_iterator; const size_t size = (1ul << 31) + 1ul; @@ -1169,7 +1173,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) unsigned int seed_value = rand(); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); - // Create test_utils::counting_iterator with random starting point + // Create rocprim::counting_iterator with random starting point InputIterator input_begin(test_utils::get_random_value(0, 200, seed_value)); T initial_value = test_utils::get_random_value(1, 10, seed_value); @@ -1187,7 +1191,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) temp_storage_size_bytes, input_begin, output_it, - test_utils::plus{}, + ::hipcub::Sum(), initial_value, size, stream)); @@ -1204,7 +1208,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) temp_storage_size_bytes, input_begin, output_it, - test_utils::plus{}, + ::hipcub::Sum(), initial_value, size, stream)); @@ -1254,7 +1258,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanFuture) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = test_utils::transform_iterator>; + using IteratorType = rocprim::transform_iterator, acc_type>; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then // as all conversions in the tests are to more precise types, @@ -1332,11 +1336,12 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanFuture) const auto future_initial_value = hipcub::FutureValue{d_initial_value}; // Check the provided aliases to be correct at compile-time - static_assert(std::is_same_v, - "The futures value type is expected to be U"); + static_assert( + std::is_same::value, + "The futures value type is expected to be U"); static_assert( - std::is_same_v, + std::is_same::value, "The futures iterator type is expected to be U*"); // temp storage diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp index c70e19a3aaeb..470410994aed 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp @@ -104,11 +104,11 @@ inline void sort_keys() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; @@ -255,11 +255,11 @@ inline void sort_keys_empty_data() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets(2); @@ -387,11 +387,10 @@ inline void sort_keys_large_segments() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets(3); @@ -532,11 +531,11 @@ inline void sort_keys_unspecified_ranges() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector begin_offsets; @@ -715,11 +714,11 @@ inline void sort_pairs() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; @@ -912,11 +911,11 @@ inline void sort_pairs_unspecified_ranges() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector values_input(size); @@ -1138,11 +1137,11 @@ inline void sort_keys_double_buffer() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; @@ -1295,11 +1294,11 @@ inline void sort_pairs_double_buffer() } else { - keys_input = test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp index e1e80ffcef38..3d5c02575b84 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,11 +28,11 @@ // hipcub API #include -#include +#include template, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1>; +using Params1 + = ::testing::Types, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1>; TYPED_TEST_SUITE(HipcubDeviceSegmentedReduceOp, Params1); @@ -125,7 +125,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduceOp, Reduce) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = _HIPCUB_STD::min(size, offset + segment_length); + const size_t end = std::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -278,7 +278,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Sum) using input_type = typename TestFixture::params::input_type; using output_type = typename TestFixture::params::output_type; - using reduce_op_type = typename test_utils::plus; + using reduce_op_type = typename hipcub::Sum; using result_type = output_type; using offset_type = unsigned int; @@ -324,7 +324,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Sum) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = _HIPCUB_STD::min(size, offset + segment_length); + const size_t end = std::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -435,11 +435,11 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Min) using input_type = typename TestFixture::params::input_type; using output_type = typename TestFixture::params::output_type; - using reduce_op_type = typename test_utils::minimum; + using reduce_op_type = typename hipcub::Min; using result_type = output_type; using offset_type = unsigned int; - constexpr input_type init = _HIPCUB_STD::numeric_limits::max(); + constexpr input_type init = std::numeric_limits::max(); reduce_op_type reduce_op; std::random_device rd; @@ -481,7 +481,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Min) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = _HIPCUB_STD::min(size, offset + segment_length); + const size_t end = std::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -592,11 +592,11 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Max) using input_type = typename TestFixture::params::input_type; using output_type = typename TestFixture::params::output_type; - using reduce_op_type = typename test_utils::maximum; + using reduce_op_type = typename hipcub::Max; using result_type = output_type; using offset_type = unsigned int; - constexpr input_type init = _HIPCUB_STD::numeric_limits::lowest(); + constexpr input_type init = std::numeric_limits::lowest(); reduce_op_type reduce_op; std::random_device rd; @@ -638,7 +638,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Max) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = _HIPCUB_STD::min(size, offset + segment_length); + const size_t end = std::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -752,14 +752,14 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Max) struct ArgMinDispatch { template - auto operator()(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream) const + auto operator()(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream) const { return hipcub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, @@ -775,14 +775,14 @@ struct ArgMinDispatch struct ArgMaxDispatch { template - auto operator()(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - _HIPCUB_STD::int64_t num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream) const + auto operator()(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + int num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream) const { return hipcub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, @@ -848,7 +848,7 @@ void test_argminmax(typename TestFixture::params::input_type empty_value) offsets.push_back(offset); Iterator x(&values_input[offset]); - const size_t end = _HIPCUB_STD::min(size, offset + segment_length); + const size_t end = std::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); if(offset < end) { @@ -1144,8 +1144,8 @@ TEST(HipcubDeviceSegmentedReduceLargeIndicesTests, LargeIndices) using T = size_t; using input_type = T; using output_type = T; - using IteratorType = test_utils::counting_iterator; - using reduce_op_type = typename test_utils::plus; + using IteratorType = rocprim::counting_iterator; + using reduce_op_type = typename hipcub::Sum; using offset_type = T; const input_type init = input_type(0); @@ -1186,7 +1186,7 @@ TEST(HipcubDeviceSegmentedReduceLargeIndicesTests, LargeIndices) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const offset_type end = _HIPCUB_STD::min(size, offset + segment_length); + const offset_type end = std::min(size, offset + segment_length); output_type aggregate = init; aggregate = reduce_op(aggregate, gauss_sum(end) - gauss_sum(offset)); aggregates_expected.push_back(aggregate); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp index 4efa95241798..b52f4290bdc6 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -109,11 +109,10 @@ inline void generate_input_data(std::vector& keys_input, } else { - keys_input - = test_utils::get_random_data(size, - _HIPCUB_STD::numeric_limits::min(), - _HIPCUB_STD::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data(size, + std::numeric_limits::min(), + std::numeric_limits::max(), + seed_value + seed_value_addition); } offsets.clear(); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp index e999514554da..aca4048a79e1 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp @@ -24,6 +24,8 @@ // hipcub API #include +#include +#include #include "single_index_iterator.hpp" #include "test_utils_bfloat16.hpp" @@ -73,7 +75,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) using U = typename TestFixture::output_type; using F = typename TestFixture::flag_type; - constexpr bool inplace = std::is_same_v; + constexpr bool inplace = std::is_same::value; hipStream_t stream = 0; // default if(TestFixture::use_graphs) @@ -109,7 +111,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) unsigned int* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(F))); - if constexpr(!inplace) + if HIPCUB_IF_CONSTEXPR(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); } @@ -133,7 +135,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceSelect::Flagged(d_temp_storage, temp_storage_size_bytes, @@ -190,7 +192,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) // Check if output values are as expected std::vector output(input.size()); - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -242,8 +244,8 @@ TEST(HipcubDeviceSelectTests, FlagNormalization) for(size_t size : test_utils::get_sizes(seed_value)) { SCOPED_TRACE(testing::Message() << "with size= " << size); - test_utils::counting_iterator d_input(0); - test_utils::counting_iterator d_flags(1); + rocprim::counting_iterator d_input(0); + rocprim::counting_iterator d_flags(1); U* d_output; unsigned int* d_selected_count_output; @@ -335,7 +337,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) using T = typename TestFixture::input_type; using U = typename TestFixture::output_type; - constexpr bool inplace = std::is_same_v; + constexpr bool inplace = std::is_same::value; hipStream_t stream = 0; // default if(TestFixture::use_graphs) @@ -367,7 +369,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) U* d_output; unsigned int* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if constexpr(!inplace) + if HIPCUB_IF_CONSTEXPR(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); } @@ -389,7 +391,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceSelect::If(d_temp_storage, temp_storage_size_bytes, @@ -446,7 +448,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) // Check if output values are as expected std::vector output(input.size()); - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -490,7 +492,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) using U = typename TestFixture::output_type; using F = typename TestFixture::flag_type; - constexpr bool inplace = std::is_same_v; + constexpr bool inplace = std::is_same::value; hipStream_t stream = 0; // default if(TestFixture::use_graphs) @@ -528,7 +530,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) unsigned int* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(F))); - if constexpr(!inplace) + if HIPCUB_IF_CONSTEXPR(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); } @@ -552,7 +554,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipcub::DeviceSelect::FlaggedIf(d_temp_storage, temp_storage_size_bytes, @@ -611,7 +613,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) // Check if output values are as expected std::vector output(input.size()); - if constexpr(inplace) + if HIPCUB_IF_CONSTEXPR(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -689,7 +691,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Unique) test_utils::host_inclusive_scan(input01.begin(), input01.end(), input.begin(), - test_utils::plus{}); + hipcub::Sum()); } // Allocate and copy to device @@ -805,11 +807,9 @@ TEST(HipcubDeviceSelectTests, UniqueDiscardOutputIterator) for(size_t size : test_utils::get_sizes(seed_value)) { SCOPED_TRACE(testing::Message() << "with size= " << size); - test_utils::counting_iterator d_input(0); - - auto d_output = test_utils::make_discard_iterator(); - - size_t* d_selected_count_output; + rocprim::counting_iterator d_input(0); + rocprim::discard_iterator d_output; + size_t* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper((&d_selected_count_output), sizeof(size_t))); @@ -902,7 +902,7 @@ TEST_P(HipcubDeviceSelectLargeIndicesTests, LargeIndicesSelectOp) #endif // Generate data - test_utils::counting_iterator d_input(0); + rocprim::counting_iterator d_input(0); U* d_output; selected_count_type* d_selected_count_output; selected_count_type expected_output_size = selected_size; @@ -974,7 +974,7 @@ TEST_P(HipcubDeviceSelectLargeIndicesTests, LargeIndicesSelectOp) template, DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, DeviceUniqueByKeyParams, DeviceUniqueByKeyParams, test_utils::custom_test_type>, - DeviceUniqueByKeyParams>; + DeviceUniqueByKeyParams>; TYPED_TEST_SUITE(HipcubDeviceUniqueByKeyTests, HipcubDeviceUniqueByKeyTestsParams); @@ -1069,7 +1069,7 @@ TYPED_TEST(HipcubDeviceUniqueByKeyTests, UniqueByKey) test_utils::host_inclusive_scan(input01.begin(), input01.end(), input_keys.begin(), - test_utils::plus{}); + hipcub::Sum()); } const auto input_values @@ -1244,8 +1244,8 @@ TEST(HipcubDeviceUniqueByKeyTests, LargeIndicesUniqueByKey) = (size + TestUniqueEqualityOp::segment - 1) / TestUniqueEqualityOp::segment; const size_t output_index = selected_count - 1; const size_t input_index = output_index * TestUniqueEqualityOp::segment; - test_utils::counting_iterator d_keys_input(0); - test_utils::counting_iterator d_values_input(123); + rocprim::counting_iterator d_keys_input(0); + rocprim::counting_iterator d_values_input(123); key_type* d_keys_output; value_type* d_values_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, sizeof(*d_keys_output))); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_spmv.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_spmv.cpp new file mode 100644 index 000000000000..5b1459954202 --- /dev/null +++ b/projects/hipcub/test/hipcub/test_hipcub_device_spmv.cpp @@ -0,0 +1,292 @@ +// MIT License +// +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "experimental/sparse_matrix.hpp" + +#include +#include + +#include "common_test_header.hpp" +#include "test_utils_assertions.hpp" + +hipcub::CachingDeviceAllocator g_allocator; + +static constexpr float alpha_const = 1.0f; +static constexpr float beta_const = 0.0f; + +// Params for tests +template +struct DeviceSpmvParams +{ + using value_type = Type; + static constexpr int32_t grid_2d = Grid2D; + static constexpr int32_t grid_3d = Grid3D; + static constexpr int32_t wheel = Wheel; + static constexpr int32_t dense = Dense; + static constexpr bool use_graphs = UseGraphs; +}; + +// --------------------------------------------------------- +// Test for scan ops taking single input value +// --------------------------------------------------------- + +template +class HipcubDeviceSpmvTests : public ::testing::Test +{ +public: + using value_type = typename Params::value_type; + static constexpr int32_t grid_2d = Params::grid_2d; + static constexpr int32_t grid_3d = Params::grid_3d; + static constexpr int32_t wheel = Params::wheel; + static constexpr int32_t dense = Params::dense; + static constexpr bool use_graphs = Params::use_graphs; +}; + +using HipcubDeviceSpmvTestsParams = ::testing::Types, + DeviceSpmvParams>; + +template +static void generate_matrix(CooMatrix& coo_matrix, + int32_t grid2d, + int32_t grid3d, + int32_t wheel, + int32_t dense) +{ + if(grid2d > 0) + { + // Generate 2D lattice + coo_matrix.InitGrid2d(grid2d, false); + } + else if(grid3d > 0) + { + // Generate 3D lattice + coo_matrix.InitGrid3d(grid3d, false); + } + else if(wheel > 0) + { + // Generate wheel graph + coo_matrix.InitWheel(wheel); + } + else if(dense > 0) + { +#if 0 + // Generate dense graph + OffsetType size = 1 << 24; // 16M nnz + args.GetCmdLineArgument("size", size); + + OffsetType rows = size / dense; + printf("dense_%d_x_%d, ", rows, dense); fflush(stdout); + coo_matrix.InitDense(rows, dense); +#endif + } +} + +template +void SpmvGold(CsrMatrix& a, + const T* vector_x, + const T* vector_y_in, + T* vector_y_out, + T alpha, + T beta) +{ + for(OffsetType row = 0; row < a.num_rows; ++row) + { + T partial = beta * vector_y_in[row]; + for(OffsetType offset = a.row_offsets[row]; offset < a.row_offsets[row + 1]; ++offset) + { + partial += alpha * a.values[offset] * vector_x[a.column_indices[offset]]; + } + vector_y_out[row] = partial; + } +} + +TYPED_TEST_SUITE(HipcubDeviceSpmvTests, HipcubDeviceSpmvTestsParams); + +TYPED_TEST(HipcubDeviceSpmvTests, Spmv) +{ + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + using T = typename TestFixture::value_type; + using OffsetType = int32_t; + constexpr int32_t grid_2d = TestFixture::grid_2d; + constexpr int32_t grid_3d = TestFixture::grid_3d; + constexpr int32_t wheel = TestFixture::wheel; + constexpr int32_t dense = TestFixture::dense; + + hipStream_t stream = 0; // default + if(TestFixture::use_graphs) + { + // Default stream does not support hipGraph stream capture, so create one + HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); + } + + CooMatrix coo_matrix; + generate_matrix(coo_matrix, grid_2d, grid_3d, wheel, dense); + + // Convert to CSR + CsrMatrix csr_matrix; + csr_matrix.FromCoo(coo_matrix); + + // Allocate input and output vectors + T* vector_x = new T[csr_matrix.num_cols]; + T* vector_y_in = new T[csr_matrix.num_rows]; + T* vector_y_out = new T[csr_matrix.num_rows]; + + for(int col = 0; col < csr_matrix.num_cols; ++col) + vector_x[col] = 1.0; + + for(int row = 0; row < csr_matrix.num_rows; ++row) + vector_y_in[row] = 1.0; + + // Compute reference answer + SpmvGold(csr_matrix, vector_x, vector_y_in, vector_y_out, alpha_const, beta_const); + + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + // Allocate and initialize GPU problem + hipcub::DeviceSpmv::SpmvParams params{}; + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + + HIP_CHECK( + g_allocator.DeviceAllocate((void**)¶ms.d_values, sizeof(T) * csr_matrix.num_nonzeros)); + HIP_CHECK(g_allocator.DeviceAllocate((void**)¶ms.d_row_end_offsets, + sizeof(OffsetType) * (csr_matrix.num_rows + 1))); + HIP_CHECK(g_allocator.DeviceAllocate((void**)¶ms.d_column_indices, + sizeof(OffsetType) * csr_matrix.num_nonzeros)); + HIP_CHECK( + g_allocator.DeviceAllocate((void**)¶ms.d_vector_x, sizeof(T) * csr_matrix.num_cols)); + HIP_CHECK( + g_allocator.DeviceAllocate((void**)¶ms.d_vector_y, sizeof(T) * csr_matrix.num_rows)); + + params.num_rows = csr_matrix.num_rows; + params.num_cols = csr_matrix.num_cols; + params.num_nonzeros = csr_matrix.num_nonzeros; + params.alpha = alpha_const; + params.beta = beta_const; + + HIP_CHECK(hipMemcpy(params.d_values, + csr_matrix.values, + sizeof(T) * csr_matrix.num_nonzeros, + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(params.d_row_end_offsets, + csr_matrix.row_offsets, + sizeof(OffsetType) * (csr_matrix.num_rows + 1), + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(params.d_column_indices, + csr_matrix.column_indices, + sizeof(OffsetType) * csr_matrix.num_nonzeros, + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(params.d_vector_x, + vector_x, + sizeof(T) * csr_matrix.num_cols, + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(params.d_vector_y, + vector_y_in, + sizeof(T) * csr_matrix.num_rows, + hipMemcpyHostToDevice)); + + // Allocate temporary storage + size_t temp_storage_bytes = 0; + void* d_temp_storage = nullptr; + + // Get amount of temporary storage needed + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, + temp_storage_bytes, + params.d_values, + params.d_row_end_offsets, + params.d_column_indices, + params.d_vector_x, + params.d_vector_y, + params.num_rows, + params.num_cols, + params.num_nonzeros, + stream)); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + + // Allocate + //HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes); + HIP_CHECK(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + test_utils::GraphHelper gHelper; + if(TestFixture::use_graphs) + gHelper.startStreamCapture(stream); + + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, + temp_storage_bytes, + params.d_values, + params.d_row_end_offsets, + params.d_column_indices, + params.d_vector_x, + params.d_vector_y, + params.num_rows, + params.num_cols, + params.num_nonzeros, + stream)); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + + if(TestFixture::use_graphs) + gHelper.createAndLaunchGraph(stream); + + HIP_CHECK(hipMemcpy(vector_y_in, + params.d_vector_y, + sizeof(T) * params.num_rows, + hipMemcpyDeviceToHost)); + + HIP_CHECK(hipPeekAtLastError()); + HIP_CHECK(hipDeviceSynchronize()); + + const auto max_row_len = csr_matrix.num_cols * csr_matrix.num_rows; + const float diff = max_row_len * test_utils::precision::value; + + for(int32_t i = 0; i < csr_matrix.num_rows; i++) + { + ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(vector_y_in[i], vector_y_out[i], diff)) + << "where index = " << i; + } + + if(TestFixture::use_graphs) + { + gHelper.cleanupGraphHelper(); + HIP_CHECK(hipStreamDestroy(stream)); + } + + // De-allocate input and output vectors + delete[] vector_x; + delete[] vector_y_in; + delete[] vector_y_out; + + HIP_CHECK(g_allocator.DeviceFree(params.d_values)); + HIP_CHECK(g_allocator.DeviceFree(params.d_row_end_offsets)); + HIP_CHECK(g_allocator.DeviceFree(params.d_column_indices)); + HIP_CHECK(g_allocator.DeviceFree(params.d_vector_x)); + HIP_CHECK(g_allocator.DeviceFree(params.d_vector_y)); + HIP_CHECK(g_allocator.DeviceFree(d_temp_storage)); +} diff --git a/projects/hipcub/test/hipcub/test_hipcub_grid.cpp b/projects/hipcub/test/hipcub/test_hipcub_grid.cpp index e9f38a585091..b7ff8715c984 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_grid.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_grid.cpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2019-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2019-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,10 +30,82 @@ #include "common_test_header.hpp" #include +#include +#include #include #include +#if defined(__HIP_PLATFORM_NVIDIA__) +_CCCL_SUPPRESS_DEPRECATED_PUSH +#else +HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +#endif +__global__ +void KernelGridBarrier(hipcub::GridBarrier global_barrier, int iterations) +#if defined(__HIP_PLATFORM_NVIDIA__) + _CCCL_SUPPRESS_DEPRECATED_POP +#else + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +#endif +{ + for (int i = 0; i < iterations; i++) + { + global_barrier.Sync(); + } +} + +TEST(HipcubGridTests, GridBarrier) +{ + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + constexpr int32_t block_size = 256; + // NOTE increasing iterations will cause huge latency for tests + constexpr int32_t iterations = 3; + int32_t grid_size = -1; + + int32_t sm_count; + int32_t max_block_threads; + int32_t max_sm_occupancy; + + HIP_CHECK(hipDeviceGetAttribute(&sm_count, hipDeviceAttributeMultiprocessorCount, device_id)); + HIP_CHECK(hipDeviceGetAttribute(&max_block_threads, hipDeviceAttributeMaxThreadsPerBlock, device_id)); + + HIP_CHECK(hipOccupancyMaxActiveBlocksPerMultiprocessor( + &max_sm_occupancy, + KernelGridBarrier, + HIPCUB_HOST_WARP_THREADS, + 0)); + + int32_t occupancy = std::min((max_block_threads / block_size), max_sm_occupancy); + + if (grid_size == -1) + { + grid_size = occupancy * sm_count; + } + else + { + occupancy = grid_size / sm_count; + } +#if defined(__HIP_PLATFORM_NVIDIA__) + _CCCL_SUPPRESS_DEPRECATED_PUSH +#else + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH +#endif + hipcub::GridBarrierLifetime global_barrier; +#if defined(__HIP_PLATFORM_NVIDIA__) + _CCCL_SUPPRESS_DEPRECATED_POP +#else + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +#endif + HIP_CHECK(global_barrier.Setup(grid_size)); + + KernelGridBarrier<<>>(global_barrier, iterations); + HIP_CHECK(hipGetLastError()); +} + template< int32_t BlockSize, class T, @@ -57,7 +129,7 @@ __global__ void KernelGridEvenShare( T value = device_output[index]; - value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}); + value = breduce_t(temp_storage).Reduce(value, hipcub::Sum()); if(hipThreadIdx_x == 0) { device_output_reductions[hipBlockIdx_x] = value; @@ -178,8 +250,7 @@ __global__ void KernelGridQueue( int32_t index = block_tile_index * BlockSize + hipThreadIdx_x; T value = device_output[index]; - - value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}); + value = breduce_t(temp_storage).Reduce(value, hipcub::Sum()); if(hipThreadIdx_x == 0) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp b/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp index 21687e800694..c3ce6e599575 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,10 @@ #include #include #include +#include +#include #include +#include #include @@ -230,7 +233,7 @@ TYPED_TEST(HipcubIteratorTests, TestConstant) HIP_CHECK(hipSetDevice(device_id)); using T = typename TestFixture::input_type; - using IteratorType = test_utils::constant_iterator; + using IteratorType = rocprim::constant_iterator; constexpr uint32_t array_size = 8; std::vector h_reference(array_size); @@ -257,7 +260,7 @@ TYPED_TEST(HipcubIteratorTests, TestCounting) HIP_CHECK(hipSetDevice(device_id)); using T = typename TestFixture::input_type; - using IteratorType = test_utils::counting_iterator; + using IteratorType = rocprim::counting_iterator; constexpr uint32_t array_size = 8; std::vector h_reference(array_size); @@ -289,7 +292,7 @@ TYPED_TEST(HipcubIteratorTests, TestTransform) using T = typename TestFixture::input_type; using CastT = typename TestFixture::input_type; - using IteratorType = test_utils::transform_iterator>; + using IteratorType = rocprim::transform_iterator, T>; constexpr int TEST_VALUES = 11000; std::vector h_data(TEST_VALUES); @@ -543,12 +546,11 @@ TYPED_TEST(HipcubIteratorTests, TestTexTransform) HIP_CHECK(d_tex_itr.BindTexture(d_data, sizeof(T) * TEST_VALUES)); // Create transform iterator - test_utils::transform_iterator> xform_itr(d_tex_itr, + rocprim::transform_iterator, T> xform_itr(d_tex_itr, op); - iterator_test_function>>( - xform_itr, - h_reference); + iterator_test_function, T>, + T>(xform_itr, h_reference); HIP_CHECK(g_allocator.DeviceFree(d_data)); } } diff --git a/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp b/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp index 7bb09a7ba3b2..f6f21f26c5c5 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -25,6 +25,7 @@ #include #include +#include #include @@ -33,7 +34,7 @@ #include #include -template +template struct custom_key_value_pair_op { using type = hipcub::KeyValuePair; @@ -134,7 +135,7 @@ static void PrefixKernel(TileState tile_state, T* d_input, T* d_output) template, int BlockSize = 64, - typename ScanOp = test_utils::plus> + typename ScanOp = hipcub::Sum> struct SinglePassScanRunner { void run(int num_items, T* d_input, T* d_output) @@ -165,7 +166,7 @@ struct custom_scan_tile_state : hipcub::ScanTileState template, - typename ScanOp = test_utils::plus> + typename ScanOp = hipcub::Sum> struct SinglePassScanParams { using type = T; @@ -271,7 +272,7 @@ static void RunningPrefixKernel(T* d_input, T* d_output) prefix_type prefix(T(), ScanOp{}); - _CCCL_PRAGMA_UNROLL_FULL() +#pragma unroll for(int i = 0; i < num_items; ++i) { T value = d_input[i]; @@ -280,7 +281,7 @@ static void RunningPrefixKernel(T* d_input, T* d_output) } } -template +template struct RunningPrefixRunner { void run(T* d_input, T* d_output) diff --git a/projects/hipcub/test/hipcub/test_hipcub_thread.cpp b/projects/hipcub/test/hipcub/test_hipcub_thread.cpp index 0cc347d7aef0..d47f85746e6f 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_thread.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_thread.cpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -484,16 +484,8 @@ void thread_reduce_kernel(Type* const device_input, Type* device_output) { size_t input_index = (hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x) * Length; size_t output_index = (hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x) * Length; - - // Load into a local array - Type values[Length]; -#pragma unroll - for(int i = 0; i < Length; i++) - { - values[i] = device_input[input_index + i]; - } - - device_output[output_index] = hipcub::ThreadReduce(values, sum_op()); + device_output[output_index] + = hipcub::ThreadReduce(&device_input[input_index], sum_op()); } TYPED_TEST(HipcubThreadOperationTests, Reduction) diff --git a/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp b/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp index db3b16407e37..b024c5267112 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,21 +24,18 @@ #include "test_utils_assertions.hpp" #include "test_utils_data_generation.hpp" -#include "test_utils_functional.hpp" #include "test_utils_thread_operators.hpp" #include #include #include +#include #include #include -#include #include #include #include -#include _HIPCUB_STD_INCLUDE(functional) - template struct ThreadOperatorsParams { @@ -81,6 +78,217 @@ TYPED_TEST_SUITE(HipcubThreadOperatorsTests, ThreadOperatorsParameters); // Commutative operators tests. +/// \brief Shared code for equality/inequality operators. +template +void equality_op_test(ScanOpT op, bool equality) +{ + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + // Generate random input value. + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + const InputT input_val + = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; + + OutputT output_val{}; + + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(input_val, input_val), equality)); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(output_val, output_val), equality)); + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(output_val, input_val), !equality)); + + output_val = OutputT(input_val); + + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(output_val, input_val), equality)); + } +} + +TYPED_TEST(HipcubThreadOperatorsTests, Equality) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + + using Equality = typename EqualitySelector::type; + Equality op{}; + + equality_op_test(op, true); +} + +TYPED_TEST(HipcubThreadOperatorsTests, Inequality) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + + using Inequality = typename EqualitySelector::type; + Inequality op{}; + + equality_op_test(op, false); +} + +TYPED_TEST(HipcubThreadOperatorsTests, InequalityWrapper) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + + using Equality = typename EqualitySelector::type; + Equality wrapped_op{}; + hipcub::InequalityWrapper op{wrapped_op}; + + equality_op_test(op, false); +} + +/// \brief Shared code for algebraic operators. +template +void algebraic_op_test(const InputT input_val, OutputT init_val) +{ + using accum_type = hipcub::detail::accumulator_t; + + ScanOpT op{}; + + accum_type output_val = init_val; + + // Check result. + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(init_val, input_val), output_val)); + + // Check return type. + ASSERT_NO_FATAL_FAILURE(test_utils::assert_type(op(init_val, input_val), output_val)); +} + +TYPED_TEST(HipcubThreadOperatorsTests, Sum) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + using Sum = typename AlgebraicSelector::type; + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + // Generate random initial value. + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + output_type init_val + = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; + + algebraic_op_test(input_type{}, init_val); + } +} + +TYPED_TEST(HipcubThreadOperatorsTests, Difference) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + using Difference = + typename AlgebraicSelector::type; + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + // Generate random initial value. + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + output_type init_val + = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; + + algebraic_op_test(input_type{}, init_val); + } +} + +// Division operator is not defined for custom_test_type. +template +class HipcubDivisionOperatorTests : public ::testing::Test +{ +public: + using input_type = typename Params::input_type; + using output_type = typename Params::output_type; +}; + +using DivisionOperatorParameters = ::testing::Types< + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams, + ThreadOperatorsParams +#ifdef __HIP_PLATFORM_AMD__ + , + ThreadOperatorsParams, // Doesn't work on NVIDIA / CUB + ThreadOperatorsParams // Doesn't work on NVIDIA / CUB +#endif + >; +TYPED_TEST_SUITE(HipcubDivisionOperatorTests, DivisionOperatorParameters); + +TYPED_TEST(HipcubDivisionOperatorTests, Division) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + using Division = typename AlgebraicSelector::type; + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + // Generate random input value. + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + input_type input_val + = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; + + algebraic_op_test(input_val, output_type{}); + } +} + +/// \brief Shared code for min/max operators. +template +void minmax_op_test(bool is_max) +{ + ScanOpT op{}; + + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + { + // Generate random initial and input values. + unsigned int seed_value + = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; + SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); + OutputT init_val = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; + InputT input_val = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; + + AccumT output_val + = is_max ? test_utils::max(init_val, input_val) : test_utils::min(init_val, input_val); + + // Check result. + ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(init_val, input_val), output_val)); + + // Check return type. + ASSERT_NO_FATAL_FAILURE(test_utils::assert_type(op(init_val, input_val), output_val)); + } +} + +TYPED_TEST(HipcubThreadOperatorsTests, Max) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + using accum_type = typename std::common_type::type; + using Max = typename MaxSelector::type; + + minmax_op_test(true); +} + +TYPED_TEST(HipcubThreadOperatorsTests, Min) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + using accum_type = typename std::common_type::type; + using Min = typename MinSelector::type; + + minmax_op_test(false); +} + /// \brief Shared code for ArgMin/ArgMax operators. template void arg_op_test(bool is_max) @@ -250,8 +458,8 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, SwizzleScanOp) std::iota(h_input.begin(), h_input.end(), static_cast(1)); // Scan function: SwizzleScanOp. - test_utils::plus sum_op{}; - hipcub::SwizzleScanOp scan_op(sum_op); + hipcub::Sum sum_op{}; + hipcub::SwizzleScanOp scan_op(sum_op); // Calculate expected results on host. std::vector h_expected(input_size); @@ -295,15 +503,15 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceBySegmentOp) } // Reduce and scan operators. - test_utils::plus sum_op{}; - hipcub::ReduceBySegmentOp op(sum_op); + hipcub::Sum sum_op{}; + hipcub::ReduceBySegmentOp op(sum_op); // Calculate expected results on host. std::vector expected{}; pair_type init(0, 0); for(size_t offset = 0; offset < input_size; offset += segment_size) { - const size_t end = _HIPCUB_STD::min(input_size, offset + segment_size); + const size_t end = std::min(input_size, offset + segment_size); pair_type aggregate = init; for(size_t i = offset; i < end; ++i) { @@ -320,7 +528,7 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceBySegmentOp) std::vector output{}; for(size_t offset = 0; offset < input_size; offset += segment_size) { - const size_t end = _HIPCUB_STD::min(input_size, offset + segment_size); + const size_t end = std::min(input_size, offset + segment_size); pair_type aggregate = init; for(size_t i = offset; i < end; ++i) { @@ -376,8 +584,8 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceByKeyOp) } // Reduce operators. - test_utils::plus sum_op; - hipcub::ReduceByKeyOp op{}; + hipcub::Sum sum_op; + hipcub::ReduceByKeyOp op{}; // Calculate output on host. std::vector h_output(h_unique_keys); @@ -488,6 +696,44 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceByKeyOp) } } +TYPED_TEST(HipcubNCThreadOperatorsTests, BinaryFlip) +{ + using input_type = typename TestFixture::input_type; + using output_type = typename TestFixture::output_type; + + const std::vector sizes = get_sizes(); + for(auto input_size : sizes) + { + // Generate data. + std::vector h_input(input_size); + std::iota(h_input.begin(), h_input.end(), static_cast(1)); + + // Scan function: BinaryFlip. + hipcub::Sum sum_op{}; + hipcub::BinaryFlip scan_op(sum_op); + + // Calculate expected results on host. + std::vector h_expected{}; + + // BinaryFlip's () operator is a device function, so cannot be called from the host function + // test_utils::host_inclusive_scan. We do the scan "manually". + output_type accum = h_input[0]; + h_expected.push_back(accum); + for(size_t i = 1; i < input_size; ++i) + { + // The host_inclusive_cast would do: + // + // accum = scan_op(accum, static_cast(h_input[i])); + // + // But for the BinaryFlip this is equivalent to: + accum = sum_op(static_cast(h_input[i]), accum); + h_expected.push_back(accum); + } + + scan_op_test(h_input, h_expected, scan_op, input_size); + } +} + // Unary operators tests. TYPED_TEST(HipcubNCThreadOperatorsTests, CastOp) @@ -495,8 +741,7 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, CastOp) using input_type = typename TestFixture::input_type; using output_type = typename TestFixture::output_type; using IteratorType - = test_utils::transform_iterator, output_type>; - + = rocprim::transform_iterator, output_type>; const std::vector sizes = get_sizes(); for(auto input_size : sizes) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_util_device.cpp b/projects/hipcub/test/hipcub/test_hipcub_util_device.cpp new file mode 100644 index 000000000000..34277a5ecfcb --- /dev/null +++ b/projects/hipcub/test/hipcub/test_hipcub_util_device.cpp @@ -0,0 +1,133 @@ +// MIT License +// +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "common_test_header.hpp" + +// hipcub API +#include + +template +__global__ +void alias_temporaries_kernel(T* data, size_t* temp_storage_bytes) +{ + T* allocations[10]; + size_t allocation_sizes[10] = {1, 2, 3, 5, 8, 13, 21, 34, 55, 89}; + (void) + hipcub::detail::AliasTemporaries(data, *temp_storage_bytes, allocations, allocation_sizes); +} + +TEST(HipcubUtilDevice, AliasTemporariesDevice) +{ + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + void* data = nullptr; + size_t temp_storage_bytes_host = 0; // Temporary storage on the host + size_t* device_temp_storage_bytes; + HIP_CHECK(test_common_utils::hipMallocHelper(&device_temp_storage_bytes, sizeof(size_t))); + + // First kernel call to determine required temp storage size + alias_temporaries_kernel<<<1, 1, 0, 0>>>(data, device_temp_storage_bytes); + HIP_CHECK(hipDeviceSynchronize()); + HIP_CHECK(hipGetLastError()); + + // Copy the device storage size to host + HIP_CHECK(hipMemcpy(&temp_storage_bytes_host, + device_temp_storage_bytes, + sizeof(size_t), + hipMemcpyDeviceToHost)); + ASSERT_GT(temp_storage_bytes_host, 0U); + + // Allocate the actual data buffer on the device + HIP_CHECK(test_common_utils::hipMallocHelper(&data, temp_storage_bytes_host)); + + // Second kernel call with allocated buffer + alias_temporaries_kernel<<<1, 1, 0, 0>>>(data, device_temp_storage_bytes); + HIP_CHECK(hipDeviceSynchronize()); + HIP_CHECK(hipGetLastError()); + + // Free device memory + HIP_CHECK(hipFree(device_temp_storage_bytes)); + HIP_CHECK(hipFree(data)); +} + +TEST(HipcubUtilDevice, AliasTemporariesHost) +{ + int device_id = test_common_utils::obtain_device_from_ctest(); + SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); + HIP_CHECK(hipSetDevice(device_id)); + + void* data = nullptr; + size_t temp_storage_bytes = 0; + void* allocations[10]; + size_t allocation_sizes[10] = {1, 789, 3, 5, 8, 13, 21, 257, 256, 890}; + + size_t min_size = 0; + for(unsigned int i = 0; i < 10; i++) + { + min_size += allocation_sizes[i]; + } + + // Determine storage size + HIP_CHECK( + hipcub::detail::AliasTemporaries(data, temp_storage_bytes, allocations, allocation_sizes)); + + // Should be larger or equal to the sum of all sizes. + ASSERT_GT(temp_storage_bytes, min_size - 1); + + // Allocate the actual data buffer on the device + HIP_CHECK(test_common_utils::hipMallocHelper(&data, temp_storage_bytes)); + + size_t zero_size = 0; + // Check for error if it does not fit + hipError_t error + = hipcub::detail::AliasTemporaries(data, zero_size, allocations, allocation_sizes); + test_utils::assert_eq(error, hipErrorInvalidValue); + + HIP_CHECK( + hipcub::detail::AliasTemporaries(data, temp_storage_bytes, allocations, allocation_sizes)); + + test_utils::assert_eq(data, allocations[0]); + + for(unsigned int i = 1; i < 10; i++) + { + // The allocations should be in increasing order. + ASSERT_GT(allocations[i], allocations[i - 1]); + size_t current_pointer = (size_t)allocations[i]; + size_t before_pointer = (size_t)allocations[i - 1]; + size_t distance = current_pointer - before_pointer; + + // Check if all pointer have enough space + ASSERT_GT(distance + 1, allocation_sizes[i - 1]); + } + + size_t last_pointer = (size_t)allocations[9]; + size_t start_pointer = (size_t)data; + size_t max_size = start_pointer + temp_storage_bytes; + size_t last_size = max_size - last_pointer; + + // Last size should be equal or larger then the last value in allocation_sizes + ASSERT_GT(last_size + 1, allocation_sizes[9]); + + HIP_CHECK(hipFree(data)); +} diff --git a/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp b/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp index 26ce3faa6a76..c583c0b9e39c 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -156,9 +156,9 @@ TYPED_TEST(HipcubUtilPtxTests, ShuffleUp) std::vector output(input.size()); auto src_offsets = test_utils::get_random_data( - _HIPCUB_STD::max(1, logical_warp_size / 2), + std::max(1, logical_warp_size / 2), 1U, - _HIPCUB_STD::max(1, logical_warp_size - 1), + std::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -267,9 +267,9 @@ TYPED_TEST(HipcubUtilPtxTests, ShuffleDown) std::vector output(input.size()); auto src_offsets = test_utils::get_random_data( - _HIPCUB_STD::max(1, logical_warp_size / 2), + std::max(1, logical_warp_size / 2), 1U, - _HIPCUB_STD::max(1, logical_warp_size - 1), + std::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -379,11 +379,10 @@ TYPED_TEST(HipcubUtilPtxTests, ShuffleIndex) seed_value); std::vector output(input.size()); - auto src_offsets - = test_utils::get_random_data(hardware_warp_size / logical_warp_size, - 0, - _HIPCUB_STD::max(1, logical_warp_size - 1), - seed_value + seed_value_addition); + auto src_offsets = test_utils::get_random_data(hardware_warp_size / logical_warp_size, + 0, + std::max(1, logical_warp_size - 1), + seed_value + seed_value_addition); // Calculate expected results on host std::vector expected(size, test_utils::convert_to_device(0)); @@ -482,9 +481,9 @@ TEST(HipcubUtilPtxTests, ShuffleUpCustomStruct) } auto src_offsets = test_utils::get_random_data( - _HIPCUB_STD::max(1, logical_warp_size / 2), + std::max(1, logical_warp_size / 2), 1U, - _HIPCUB_STD::max(1, logical_warp_size - 1), + std::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -593,9 +592,9 @@ TEST(HipcubUtilPtxTests, ShuffleUpCustomAlignedStruct) } auto src_offsets = test_utils::get_random_data( - _HIPCUB_STD::max(1, logical_warp_size / 2), + std::max(1, logical_warp_size / 2), 1U, - _HIPCUB_STD::max(1, logical_warp_size - 1), + std::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -671,11 +670,7 @@ __global__ void warp_id_kernel(unsigned int* output) { const unsigned int index = (hipBlockIdx_x * hipBlockDim_x) + hipThreadIdx_x; -#ifdef __HIP_PLATFORM_NVIDIA__ - output[index] = hipThreadIdx_x / warpSize; -#else - output[index] = ::rocprim::warp_id(); -#endif + output[index] = ::rocprim::warp_id(); } TEST(HipcubUtilPtxTests, WarpId) @@ -759,11 +754,7 @@ template HIPCUB_DEVICE std::enable_if_t<(HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize), TestStatus> test_warp_mask_pow_two() { -#ifdef __HIP_PLATFORM_NVIDIA__ - const unsigned int logical_warp_id = (hipThreadIdx_x % warpSize) / LogicalWarpSize; -#else const unsigned int logical_warp_id = ::rocprim::lane_id() / LogicalWarpSize; -#endif const uint64_t mask = hipcub::WarpMask(logical_warp_id); const unsigned int warp_start = logical_warp_id * LogicalWarpSize; @@ -804,11 +795,7 @@ template HIPCUB_DEVICE std::enable_if_t<(HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize), TestStatus> test_warp_mask_non_pow_two() { -#ifdef __HIP_PLATFORM_NVIDIA__ - const unsigned int logical_warp_id = (hipThreadIdx_x % warpSize) / LogicalWarpSize; -#else const unsigned int logical_warp_id = ::rocprim::lane_id() / LogicalWarpSize; -#endif const uint64_t mask = hipcub::WarpMask(logical_warp_id); for(unsigned int lane = 0; lane < LogicalWarpSize; ++lane) diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp index 25bd35263d5f..30adcdbb5733 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -133,7 +133,11 @@ __device__ auto warp_exchange_test(T* d_input, T* d_output) thread_data[i] = d_input[threadIdx.x * ItemsPerThread + i]; } - using WarpExchangeT = ::hipcub::WarpExchange; + using WarpExchangeT = ::hipcub::WarpExchange; constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; __shared__ typename WarpExchangeT::TempStorage temp_storage[warps_in_block]; const unsigned warp_id = threadIdx.x / LogicalWarpSize; @@ -218,7 +222,7 @@ std::enable_if_t> run_warp_exch input[i] = test_utils::convert_to_device(i); } std::vector expected; - if(std::is_same_v) + if(std::is_same::value) { expected = input; input = stripe_vector(input, warp_size, items_per_thread); diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp index f35adb743c9c..71cead9b29a8 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -232,14 +232,14 @@ TYPED_TEST(HipcubWarpLoadTest, WarpLoadGuarded) SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using T = typename TestFixture::params::type; - constexpr unsigned warp_size = TestFixture::params::warp_size; - constexpr ::hipcub::WarpLoadAlgorithm algorithm = TestFixture::params::algorithm; - constexpr unsigned items_per_thread = 4; - constexpr unsigned block_size = 1024; - constexpr unsigned items_count = items_per_thread * block_size; - constexpr int valid_items = warp_size / 4; - constexpr T oob_default = _HIPCUB_STD::numeric_limits::max(); + using T = typename TestFixture::params::type; + constexpr unsigned warp_size = TestFixture::params::warp_size; + constexpr ::hipcub::WarpLoadAlgorithm algorithm = TestFixture::params::algorithm; + constexpr unsigned items_per_thread = 4; + constexpr unsigned block_size = 1024; + constexpr unsigned items_count = items_per_thread * block_size; + constexpr int valid_items = warp_size / 4; + constexpr T oob_default = std::numeric_limits::max(); SKIP_IF_UNSUPPORTED_WARP_SIZE(warp_size); diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp index c5aab18cf110..9055e377558c 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -98,7 +99,7 @@ __device__ auto sort_keys_full_test(Key* keys, Compare compare_op) __shared__ typename warp_merge_sort::TempStorage storage[warps_per_block]; warp_merge_sort wsort{storage[warp_id]}; - if constexpr(Stable) + if HIPCUB_IF_CONSTEXPR(Stable) { wsort.StableSort(thread_keys, compare_op); } else @@ -156,7 +157,7 @@ __device__ auto sort_keys_values_full_test(Key* keys, Value* values, Compare com __shared__ typename warp_merge_sort::TempStorage storage[warps_per_block]; warp_merge_sort wsort{storage[warp_id]}; - if constexpr(Stable) + if HIPCUB_IF_CONSTEXPR(Stable) { wsort.StableSort(thread_keys, thread_values, compare_op); } else @@ -201,26 +202,12 @@ struct sort_last; template struct sort_last { - static constexpr T value = _HIPCUB_STD::numeric_limits::max(); + static constexpr T value = std::numeric_limits::max(); }; template struct sort_last { - static constexpr T value = _HIPCUB_STD::numeric_limits::lowest(); -}; - -template -struct sort_last> -{ - static constexpr test_utils::custom_test_type value = test_utils::custom_test_type( - _HIPCUB_STD::numeric_limits::max(), _HIPCUB_STD::numeric_limits::max()); -}; - -template -struct sort_last> -{ - static constexpr test_utils::custom_test_type value = test_utils::custom_test_type( - _HIPCUB_STD::numeric_limits::lowest(), _HIPCUB_STD::numeric_limits::lowest()); + static constexpr T value = std::numeric_limits::lowest(); }; template::value; - if constexpr(Stable) + if HIPCUB_IF_CONSTEXPR(Stable) { wsort.StableSort(thread_keys, compare, segment_size, oob_default); } else @@ -324,7 +311,7 @@ __device__ auto sort_keys_values_segmented_test(Key* keys, hipcub::LoadDirectBlocked(flat_tid, values + warp_offset, thread_values, segment_size); const Key oob_default = sort_last::value; - if constexpr(Stable) + if HIPCUB_IF_CONSTEXPR(Stable) { wsort.StableSort(thread_keys, thread_values, compare, segment_size, oob_default); } else @@ -413,15 +400,15 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeysSegmented) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::lowest(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + size, + std::numeric_limits::lowest(), + std::numeric_limits::max(), + seed_value); const auto segment_sizes = test_utils::get_random_data( num_warps, 0u, max_segment_size, ~seed_value); @@ -526,29 +513,29 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeysValuesSegmented) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::lowest(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + size, + std::numeric_limits::lowest(), + std::numeric_limits::max(), + seed_value); using value_wrapped_type = typename test_utils::inner_type::type; auto values = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::lowest(), - _HIPCUB_STD::numeric_limits::max(), - seed_value ^ (seed_value >> 1ul)); + size, + std::numeric_limits::lowest(), + std::numeric_limits::max(), + seed_value ^ (seed_value >> 1ul)); const auto segment_sizes = test_utils::get_random_data( num_warps, 0u, max_segment_size, ~seed_value); @@ -674,15 +661,15 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeys) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::lowest(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + size, + std::numeric_limits::lowest(), + std::numeric_limits::max(), + seed_value); const auto compare = typename params::compare_function{}; @@ -773,29 +760,29 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeysValues) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::lowest(), - _HIPCUB_STD::numeric_limits::max(), - seed_value); + size, + std::numeric_limits::lowest(), + std::numeric_limits::max(), + seed_value); using value_wrapped_type = typename test_utils::inner_type::type; auto values = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - _HIPCUB_STD::numeric_limits::lowest(), - _HIPCUB_STD::numeric_limits::max(), - seed_value ^ (seed_value >> 1ul)); + size, + std::numeric_limits::lowest(), + std::numeric_limits::max(), + seed_value ^ (seed_value >> 1ul)); const auto compare = typename params::compare_function{}; diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp index 0f36220cc98f..d60cb0333cc7 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -122,9 +122,7 @@ auto warp_reduce_kernel(T* device_input, T* device_output) -> using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage[warps_no]; - - auto reduce_op = test_utils::plus{}; - + auto reduce_op = hipcub::Sum(); value = wreduce_t(storage[warp_id]).Reduce(value, reduce_op); if (hipThreadIdx_x % LogicalWarpSize == 0) @@ -282,9 +280,7 @@ auto warp_reduce_valid_kernel(T* device_input, T* device_output, const int valid using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage[warps_no]; - - auto reduce_op = test_utils::plus{}; - + auto reduce_op = hipcub::Sum(); value = wreduce_t(storage[warp_id]).Reduce(value, reduce_op, valid); if (hipThreadIdx_x % LogicalWarpSize == 0) @@ -653,9 +649,7 @@ auto tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) -> using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage[warps_no]; - - auto reduce_op = test_utils::plus{}; - + auto reduce_op = hipcub::Sum(); value = wreduce_t(storage[warp_id]).TailSegmentedReduce(value, flag, reduce_op); output[index] = value; diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp index 4d801be1bf69..9a91412211ff 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -121,8 +121,7 @@ auto warp_inclusive_scan_kernel(T* device_input, T* device_output) using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - - auto scan_op = test_utils::plus{}; + auto scan_op = hipcub::Sum(); wscan_t(storage[warp_id]).InclusiveScan(value, value, scan_op); device_output[index] = value; @@ -273,7 +272,7 @@ auto warp_inclusive_scan_initial_value_kernel(T* device_input, T* device_output, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - auto scan_op = test_utils::plus{}; + auto scan_op = hipcub::Sum(); wscan_t(storage[warp_id]).InclusiveScan(value, value, initial_value, scan_op); device_output[index] = value; @@ -446,7 +445,7 @@ auto warp_inclusive_scan_reduce_kernel(T* device_input, __shared__ typename wscan_t::TempStorage storage[warps_no]; if(hipBlockIdx_x%2 == 0) { - auto scan_op = test_utils::plus{}; + auto scan_op = hipcub::Sum(); wscan_t(storage[warp_id]).InclusiveScan(value, value, scan_op, reduction); } else @@ -631,8 +630,7 @@ auto warp_inclusive_scan_reduce_initial_value_kernel(T* device_input, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - wscan_t(storage[warp_id]) - .InclusiveScan(value, value, initial_value, test_utils::plus{}, reduction); + wscan_t(storage[warp_id]).InclusiveScan(value, value, initial_value, hipcub::Sum(), reduction); device_output[index] = value; if((hipThreadIdx_x % LogicalWarpSize) == 0) @@ -819,8 +817,7 @@ auto warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - - auto scan_op = test_utils::plus{}; + auto scan_op = hipcub::Sum(); wscan_t(storage[warp_id]).ExclusiveScan(value, value, init, scan_op); device_output[index] = value; @@ -978,8 +975,7 @@ auto warp_exclusive_scan_reduce_kernel(T* device_input, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - - auto scan_op = test_utils::plus{}; + auto scan_op = hipcub::Sum(); wscan_t(storage[warp_id]).ExclusiveScan(value, value, init, scan_op, reduction); device_output[index] = value; @@ -1173,8 +1169,7 @@ auto warp_scan_kernel(T* device_input, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - - auto scan_op = test_utils::plus{}; + auto scan_op = hipcub::Sum(); wscan_t(storage[warp_id]).Scan(input, inclusive_output, exclusive_output, init, scan_op); device_inclusive_output[index] = inclusive_output; diff --git a/projects/hipcub/test/hipcub/test_utils.hpp b/projects/hipcub/test/hipcub/test_utils.hpp index fff732ed7647..88eac81521a4 100644 --- a/projects/hipcub/test/hipcub/test_utils.hpp +++ b/projects/hipcub/test/hipcub/test_utils.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,29 +21,18 @@ #ifndef HIPCUB_TEST_TEST_UTILS_HPP_ #define HIPCUB_TEST_TEST_UTILS_HPP_ -#ifndef TEST_UTILS_INCLUDE_GUARD +#ifndef TEST_UTILS_INCLUDE_GAURD #error test_utils.hpp must ONLY be included by common_test_header.hpp. Please include common_test_header.hpp instead. #endif // hipCUB API #ifdef __HIP_PLATFORM_AMD__ - #include - #include - #include - #include - #include + #include #elif defined(__HIP_PLATFORM_NVIDIA__) - #include - #include - #include #include - #include - #include - #include + #include #endif -#include - #include "test_utils_assertions.hpp" #include "test_utils_bfloat16.hpp" #include "test_utils_custom_test_types.hpp" @@ -56,8 +45,6 @@ // Seed values #include "test_seed.hpp" -#include -#include #include namespace test_utils @@ -180,7 +167,7 @@ OutputIt host_inclusive_scan_impl( template OutputIt host_inclusive_scan(InputIt first, InputIt last, OutputIt d_first, BinaryOperation op) { - using acc_type = ::hipcub::detail::it_value_t; + using acc_type = typename std::iterator_traits::value_type; return host_inclusive_scan_impl(first, last, d_first, op, acc_type{}); } @@ -191,31 +178,35 @@ OutputIt host_inclusive_scan_init( return host_inclusive_scan_impl(first, last, d_first, op, init_value); } -template< - class InputIt, - class OutputIt, - class T, - std::enable_if_t, test_utils::bfloat16> - || std::is_same_v, test_utils::half> - || std::is_same_v, float>, - bool> - = true> +template::value_type, + test_utils::bfloat16>::value + || std::is_same::value_type, + test_utils::half>::value + || std::is_same::value_type, float>::value, + bool> + = true> OutputIt host_inclusive_scan(InputIt first, InputIt last, OutputIt d_first, test_utils::plus) { using acc_type = double; return host_inclusive_scan_impl(first, last, d_first, test_utils::plus(), acc_type{}); } -template< - class InputIt, - class OutputIt, - class InitType, - class T, - std::enable_if_t, test_utils::bfloat16> - || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> - || std::is_same_v<::hipcub::detail::it_value_t, float>, - bool> - = true> +template::value_type, + test_utils::bfloat16>::value + || std::is_same::value_type, + test_utils::half>::value + || std::is_same::value_type, float>::value, + bool> + = true> OutputIt host_inclusive_scan_init( InputIt first, InputIt last, OutputIt d_first, InitType init_value, test_utils::plus) { @@ -251,20 +242,22 @@ template OutputIt host_exclusive_scan( InputIt first, InputIt last, T initial_value, OutputIt d_first, BinaryOperation op) { - using acc_type = ::hipcub::detail::it_value_t; + using acc_type = typename std::iterator_traits::value_type; return host_exclusive_scan_impl(first, last, initial_value, d_first, op, acc_type{}); } -template< - class InputIt, - class T, - class OutputIt, - class U, - std::enable_if_t, test_utils::bfloat16> - || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> - || std::is_same_v<::hipcub::detail::it_value_t, float>, - bool> - = true> +template::value_type, + test_utils::bfloat16>::value + || std::is_same::value_type, + test_utils::half>::value + || std::is_same::value_type, float>::value, + bool> + = true> OutputIt host_exclusive_scan( InputIt first, InputIt last, T initial_value, OutputIt d_first, test_utils::plus) { @@ -328,7 +321,7 @@ OutputIt host_exclusive_scan_by_key(InputIt first, BinaryOperation op, KeyCompare key_compare_op) { - using acc_type = ::hipcub::detail::it_value_t; + using acc_type = typename std::iterator_traits::value_type; return host_exclusive_scan_by_key_impl(first, last, k_first, @@ -339,18 +332,20 @@ OutputIt host_exclusive_scan_by_key(InputIt first, acc_type{}); } -template< - class InputIt, - class KeyIt, - class T, - class OutputIt, - class U, - class KeyCompare, - std::enable_if_t, test_utils::bfloat16> - || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> - || std::is_same_v<::hipcub::detail::it_value_t, float>, - bool> - = true> +template::value_type, + test_utils::bfloat16>::value + || std::is_same::value_type, + test_utils::half>::value + || std::is_same::value_type, float>::value, + bool> + = true> OutputIt host_exclusive_scan_by_key(InputIt first, InputIt last, KeyIt k_first, @@ -412,7 +407,7 @@ OutputIt host_inclusive_scan_by_key(InputIt first, BinaryOperation op, KeyCompare key_compare_op) { - using acc_type = ::hipcub::detail::it_value_t; + using acc_type = typename std::iterator_traits::value_type; return host_inclusive_scan_by_key_impl(first, last, k_first, @@ -422,17 +417,19 @@ OutputIt host_inclusive_scan_by_key(InputIt first, acc_type{}); } -template< - class InputIt, - class KeyIt, - class OutputIt, - class U, - class KeyCompare, - std::enable_if_t, test_utils::bfloat16> - || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> - || std::is_same_v<::hipcub::detail::it_value_t, float>, - bool> - = true> +template::value_type, + test_utils::bfloat16>::value + || std::is_same::value_type, + test_utils::half>::value + || std::is_same::value_type, float>::value, + bool> + = true> OutputIt host_inclusive_scan_by_key(InputIt first, InputIt last, KeyIt k_first, @@ -622,8 +619,8 @@ constexpr T get_min_warp_size(const T block_size, const T max_warp_size) } template -constexpr bool device_test_enabled_for_warp_size_v - = (HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize); +__device__ constexpr bool device_test_enabled_for_warp_size_v + = HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize; template 0 ? 1 : 0); } -#if defined(__HIP_PLATFORM_AMD__) - -template -using extents = ::hipcub::extents; - -template -struct extents_size; - -template -struct extents_size> -{ - static constexpr std::size_t value = (Dims * ... * 1); -}; - -template -using constant_iterator = ::rocprim::constant_iterator; - -template -using counting_iterator = ::rocprim::counting_iterator; - -template> -using transform_iterator = ::rocprim::transform_iterator; - -struct discard_iterator : public ::rocprim::discard_iterator -{ - using base_type = ::rocprim::discard_iterator; - using value_type = void; - using difference_type = std::ptrdiff_t; - using iterator_category = std::random_access_iterator_tag; - - using base_type::base_type; - - discard_iterator(const ::rocprim::discard_iterator& other) : base_type(other) {} -}; - -using discard_output_iterator = discard_iterator; - -inline auto make_discard_iterator() -> discard_iterator -{ - return discard_iterator(::rocprim::make_discard_iterator()); -} - -#elif defined(__HIP_PLATFORM_NVIDIA__) - -template -using extents = ::cuda::std::extents; - -template -struct extents_size; - -template -struct extents_size> -{ - static constexpr std::size_t value = (Dims * ... * 1); -}; - -template -using constant_iterator = ::cub::ConstantInputIterator; - -template -using counting_iterator = ::cub::CountingInputIterator; - -template> -using transform_iterator = ::cub::TransformInputIterator; - -template -using discard_iterator = ::cub::DiscardOutputIterator; - -template -using discard_output_iterator = ::cub::DiscardOutputIterator; - -template -inline auto make_discard_iterator() -> ::cub::DiscardOutputIterator -{ - return ::cub::DiscardOutputIterator(); -} - -#endif - } // namespace test_utils // Need for hipcub::DeviceReduce::Min/Max etc. @@ -726,17 +644,17 @@ namespace std static constexpr inline T max() { - return _HIPCUB_STD::numeric_limits::max(); + return std::numeric_limits::max(); } static constexpr inline T min() { - return _HIPCUB_STD::numeric_limits::min(); + return std::numeric_limits::min(); } static constexpr inline T lowest() { - return _HIPCUB_STD::numeric_limits::lowest(); + return std::numeric_limits::lowest(); } }; @@ -749,17 +667,17 @@ namespace std static constexpr inline T max() { - return _HIPCUB_STD::numeric_limits::max(); + return std::numeric_limits::max(); } static constexpr inline T min() { - return _HIPCUB_STD::numeric_limits::min(); + return std::numeric_limits::min(); } static constexpr inline T lowest() { - return _HIPCUB_STD::numeric_limits::lowest(); + return std::numeric_limits::lowest(); } }; } diff --git a/projects/hipcub/test/hipcub/test_utils_assertions.hpp b/projects/hipcub/test/hipcub/test_utils_assertions.hpp index ffed4ccfdd88..08903f7ce5c7 100644 --- a/projects/hipcub/test/hipcub/test_utils_assertions.hpp +++ b/projects/hipcub/test/hipcub/test_utils_assertions.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2023 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,7 +48,7 @@ template inline void assert_eq(const std::vector& result, const std::vector& expected, const size_t max_length = SIZE_MAX) { if(max_length == SIZE_MAX || max_length > expected.size()) ASSERT_EQ(result.size(), expected.size()); - for(size_t i = 0; i < _HIPCUB_STD::min(result.size(), max_length); i++) + for(size_t i = 0; i < std::min(result.size(), max_length); i++) { if(bit_equal(result[i], expected[i])) continue; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. @@ -74,7 +74,7 @@ inline void assert_eq(const std::vector& result, const std::vector& expect inline void assert_eq(const std::vector& result, const std::vector& expected, const size_t max_length = SIZE_MAX) { if(max_length == SIZE_MAX || max_length > expected.size()) ASSERT_EQ(result.size(), expected.size()); - for(size_t i = 0; i < _HIPCUB_STD::min(result.size(), max_length); i++) + for(size_t i = 0; i < std::min(result.size(), max_length); i++) { if(bit_equal(result[i], expected[i])) continue; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. ASSERT_EQ(test_utils::native_half(result[i]), test_utils::native_half(expected[i])) << "where index = " << i; @@ -84,7 +84,7 @@ inline void assert_eq(const std::vector& result, const std::ve inline void assert_eq(const std::vector& result, const std::vector& expected, const size_t max_length = SIZE_MAX) { if(max_length == SIZE_MAX || max_length > expected.size()) ASSERT_EQ(result.size(), expected.size()); - for(size_t i = 0; i < _HIPCUB_STD::min(result.size(), max_length); i++) + for(size_t i = 0; i < std::min(result.size(), max_length); i++) { if(bit_equal(result[i], expected[i])) continue; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. ASSERT_EQ(test_utils::native_bfloat16(result[i]), test_utils::native_bfloat16(expected[i])) << "where index = " << i; @@ -137,13 +137,9 @@ inline auto assert_near(const std::vector& result, const std::vector& expe } } -template< - class T, - std::enable_if_t || std::is_same_v, - bool> - = true> -inline void - assert_near(const std::vector& result, const std::vector& expected, const float percent) +template::value || + std::is_same::value, bool> = true> +inline void assert_near(const std::vector& result, const std::vector& expected, const float percent) { ASSERT_EQ(result.size(), expected.size()); for(size_t i = 0; i < result.size(); i++) @@ -180,14 +176,9 @@ inline auto assert_near(const std::vector>& result, const st } } -template< - class T, - std::enable_if_t || std::is_same_v, - bool> - = true> -inline void assert_near(const std::vector>& result, - const std::vector>& expected, - const float percent) +template::value || + std::is_same::value, bool> = true> +inline void assert_near(const std::vector>& result, const std::vector>& expected, const float percent) { ASSERT_EQ(result.size(), expected.size()); for(size_t i = 0; i < result.size(); i++) @@ -218,11 +209,8 @@ inline auto assert_near(const T& result, const T& expected, const float) ASSERT_EQ(result, expected); } -template< - class T, - std::enable_if_t || std::is_same_v, - bool> - = true> +template::value || + std::is_same::value, bool> = true> inline void assert_near(const T& result, const T& expected, const float percent) { if(bit_equal(result, expected)) return; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. @@ -236,10 +224,8 @@ inline auto assert_near(const custom_test_type& result, const custom_test_typ { auto diff1 = std::abs(percent * expected.x); auto diff2 = std::abs(percent * expected.y); - if(!bit_equal(result.x, expected.x)) - ASSERT_NEAR(result.x, expected.x, diff1); - if(!bit_equal(result.y, expected.y)) - ASSERT_NEAR(result.y, expected.y, diff2); + if(!bit_equal(result.x, expected.x)) ASSERT_NEAR(result.x, expected.x, diff1); + if(!bit_equal(result.x, expected.x)) ASSERT_NEAR(result.y, expected.y, diff2); } template @@ -268,7 +254,7 @@ inline void assert_bit_eq(const std::vector& result, const std::vector& ex } } -#if _CCCL_HAS_INT128() +#if HIPCUB_IS_INT128_ENABLED inline void assert_bit_eq(const std::vector<__int128_t>& result, const std::vector<__int128_t>& expected) { @@ -342,7 +328,7 @@ inline void assert_bit_eq(const std::vector<__uint128_t>& result, } } } -#endif //_CCCL_HAS_INT128() +#endif //HIPCUB_IS_INT128_ENABLED /// Compile-time assertion for type equality of two objects. template @@ -350,5 +336,5 @@ inline void assert_type(ExpectedT /*obj1*/, ActualT /*obj2*/) { testing::StaticAssertTypeEq(); } -} // namespace test_utils +} #endif // HIPCUB_TEST_HIPCUB_TEST_UTILS_ASSERTIONS_HPP_ diff --git a/projects/hipcub/test/hipcub/test_utils_data_generation.hpp b/projects/hipcub/test/hipcub/test_utils_data_generation.hpp index 9a320d1f17cf..75f0010fe184 100644 --- a/projects/hipcub/test/hipcub/test_utils_data_generation.hpp +++ b/projects/hipcub/test/hipcub/test_utils_data_generation.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,15 +48,14 @@ T set_half_bits(uint16_t value) // Numeric limits which also supports custom_test_type classes template -struct numeric_limits : _HIPCUB_STD::numeric_limits +struct numeric_limits : std::numeric_limits {}; template<> -struct numeric_limits : public _HIPCUB_STD::numeric_limits +struct numeric_limits : public std::numeric_limits { public: using T = test_utils::half; - static constexpr bool is_specialized = true; static inline T min() { return T(0.00006104f); @@ -75,11 +74,11 @@ struct numeric_limits : public _HIPCUB_STD::numeric_limits::quiet_NaN()); + return T(std::numeric_limits::quiet_NaN()); }; static inline T signaling_NaN() { - return T(_HIPCUB_STD::numeric_limits::signaling_NaN()); + return T(std::numeric_limits::signaling_NaN()); }; static inline T infinity_neg() { @@ -88,19 +87,17 @@ struct numeric_limits : public _HIPCUB_STD::numeric_limits -class numeric_limits - : public _HIPCUB_STD::numeric_limits +class numeric_limits : public std::numeric_limits { public: using T = test_utils::bfloat16; - static constexpr bool is_specialized = true; static inline T max() { return set_half_bits(0x7f7f); }; static inline T min() { - return T(_HIPCUB_STD::numeric_limits::min()); + return T(std::numeric_limits::min()); }; static inline T lowest() { @@ -112,11 +109,11 @@ class numeric_limits }; static inline T quiet_NaN() { - return T(_HIPCUB_STD::numeric_limits::quiet_NaN()); + return T(std::numeric_limits::quiet_NaN()); }; static inline T signaling_NaN() { - return T(_HIPCUB_STD::numeric_limits::signaling_NaN()); + return T(std::numeric_limits::signaling_NaN()); }; static inline T infinity_neg() { @@ -125,41 +122,40 @@ class numeric_limits }; template<> -class numeric_limits : public _HIPCUB_STD::numeric_limits +class numeric_limits : public std::numeric_limits { public: static inline float infinity_neg() { - return -_HIPCUB_STD::numeric_limits::infinity(); + return -std::numeric_limits::infinity(); }; }; // End of extended numeric_limits -#if _CCCL_HAS_INT128() +#if HIPCUB_IS_INT128_ENABLED template -using is_int128 = _HIPCUB_STD::is_same<__int128_t, typename std::remove_cv::type>; +using is_int128 = std::is_same<__int128_t, typename std::remove_cv::type>; template -using is_uint128 = _HIPCUB_STD::is_same<__uint128_t, typename std::remove_cv::type>; +using is_uint128 = std::is_same<__uint128_t, typename std::remove_cv::type>; #else template using is_int128 = std::false_type; template using is_uint128 = std::false_type; -#endif // _CCCL_HAS_INT128() +#endif // HIPCUB_IS_INT128_ENABLED template -using is_half = _HIPCUB_STD::is_same::type>; +using is_half = std::is_same::type>; template -using is_bfloat16 = _HIPCUB_STD::is_same::type>; +using is_bfloat16 = std::is_same::type>; template -using is_native_half - = _HIPCUB_STD::is_same::type>; +using is_native_half = std::is_same::type>; template using is_native_bfloat16 - = _HIPCUB_STD::is_same::type>; + = std::is_same::type>; template struct convert_to_native_t_impl @@ -337,12 +333,12 @@ void add_special_values(std::vector& source, int seed_value) // Actually causes problems with signed/unsigned char on Windows using clang. template struct is_valid_for_int_distribution - : std::integral_constant || std::is_same_v - || std::is_same_v || std::is_same_v - || std::is_same_v || std::is_same_v - || std::is_same_v - || std::is_same_v> + : std::integral_constant< + bool, + std::is_same::value || std::is_same::value + || std::is_same::value || std::is_same::value + || std::is_same::value || std::is_same::value + || std::is_same::value || std::is_same::value> {}; template @@ -364,8 +360,8 @@ inline auto get_random_data(size_t size, T min, T max, int seed_value) -> template inline auto get_random_data(size_t size, S min, U max, int seed_value) -> typename std::enable_if::value && !is_custom_test_type::value - && !std::is_same_v - && !std::is_same_v, + && !std::is_same::value + && !std::is_same::value, std::vector>::type { std::default_random_engine gen(seed_value); @@ -380,7 +376,7 @@ inline auto get_random_data(size_t size, S min, U max, int seed_value) -> template inline auto get_random_data(size_t size, S min, U max, int seed_value) -> - typename std::enable_if, std::vector>::type + typename std::enable_if::value, std::vector>::type { std::default_random_engine gen(seed_value); std::uniform_int_distribution distribution(static_cast(min), @@ -394,7 +390,7 @@ inline auto get_random_data(size_t size, S min, U max, int seed_value) -> template inline auto get_random_data(size_t size, S min, U max, int seed_value) -> - typename std::enable_if, std::vector>::type + typename std::enable_if::value, std::vector>::type { std::default_random_engine gen(seed_value); std::uniform_int_distribution distribution(static_cast(min), @@ -464,21 +460,16 @@ inline std::vector get_random_data01(size_t size, float p, int seed_value) std::bernoulli_distribution distribution(p); std::vector data(size); std::generate(data.begin(), - data.begin() + _HIPCUB_STD::min(size, max_random_size), + data.begin() + std::min(size, max_random_size), [&]() { return convert_to_device(distribution(gen)); }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(data.begin(), _HIPCUB_STD::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); } return data; } -// Windows HIP cannot support allocations >= 4 GiB. -#ifdef _WIN32 -template -#else template -#endif inline std::vector get_large_sizes(int seed_value) { // clang-format off @@ -489,7 +480,7 @@ inline std::vector get_large_sizes(int seed_value) // clang-format on const std::vector random_sizes = test_utils::get_random_data(2, - (size_t{1} << (MaxPow2 - 5)) + 1, + (size_t{1} << 30) + 1, (size_t{1} << MaxPow2) - 2, seed_value); sizes.insert(sizes.end(), random_sizes.begin(), random_sizes.end()); diff --git a/projects/hipcub/test/hipcub/test_utils_functional.hpp b/projects/hipcub/test/hipcub/test_utils_functional.hpp index d74214479ddf..404ad12b11c0 100644 --- a/projects/hipcub/test/hipcub/test_utils_functional.hpp +++ b/projects/hipcub/test/hipcub/test_utils_functional.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,8 +35,7 @@ namespace test_utils struct less { template - HIPCUB_HOST_DEVICE - constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const { return a < b; } @@ -45,8 +44,7 @@ struct less struct less_equal { template - HIPCUB_HOST_DEVICE - constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const { return a <= b; } @@ -55,8 +53,7 @@ struct less_equal struct greater { template - HIPCUB_HOST_DEVICE - constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const { return a > b; } @@ -65,38 +62,16 @@ struct greater struct greater_equal { template - HIPCUB_HOST_DEVICE - constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const { return a >= b; } }; -struct equal -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& a, const U& b) const - { - return a == b; - } -}; - -struct not_equal -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& a, const U& b) const - { - return a != b; - } -}; - struct plus { - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a + b) + template + HIPCUB_HOST_DEVICE inline constexpr T operator()(const T& a, const T& b) const { return a + b; } @@ -104,9 +79,8 @@ struct plus struct minus { - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a - b) + template + HIPCUB_HOST_DEVICE inline constexpr T operator()(const T& a, const T& b) const { return a - b; } @@ -114,48 +88,13 @@ struct minus struct multiplies { - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a * b) + template + HIPCUB_HOST_DEVICE inline constexpr T operator()(const T& a, const T& b) const { return a * b; } }; -struct divides -{ - template - HIPCUB_HOST_DEVICE - inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a / b) - { - return a / b; - } -}; - -struct maximum -{ - template - HIPCUB_HOST_DEVICE - auto operator()(const T& a, const U& b) const - { - using result_type = ::std::common_type_t; - result_type ra = a, rb = b; - return ra < rb ? rb : ra; - } -}; - -struct minimum -{ - template - HIPCUB_HOST_DEVICE - auto operator()(const T& a, const U& b) const - { - using result_type = ::std::common_type_t; - result_type ra = a, rb = b; - return ra < rb ? ra : rb; - } -}; - // HALF template<> HIPCUB_HOST_DEVICE inline bool diff --git a/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp b/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp index 00fe6bb8d86a..27fc65c40769 100644 --- a/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp +++ b/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -33,7 +33,6 @@ #include "test_utils_custom_test_types.hpp" #include "test_utils_half.hpp" -#include #include #include @@ -43,62 +42,40 @@ namespace test_utils namespace detail { -template -constexpr bool is_extended_int - = std::is_same_v || std::is_same_v; template clauses can be removed once - // libhipcxx is a hard depedency and test types half_t - // and bfloat_t are removed. - _HIPCUB_STD::is_integral_v || is_extended_int, - int> + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH std::enable_if_t< + hipcub::NumericTraits::CATEGORY == hipcub::SIGNED_INTEGER + || hipcub::NumericTraits::CATEGORY == hipcub::UNSIGNED_INTEGER, + int> HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP = 0> Key to_bits(const Key key) { - using Bits = typename hipcub::Traits::UnsignedBits; - static constexpr Key radix_mask_upper = EndBit == 8 * sizeof(Key) - ? static_cast(~Bits(0)) - : static_cast((Bits(1) << EndBit) - 1); + static constexpr Key radix_mask_upper + = EndBit == 8 * sizeof(Key) ? ~Key(0) : static_cast((Key(1) << EndBit) - 1); static constexpr Key radix_mask_bottom = static_cast((Key(1) << StartBit) - 1); static constexpr Key radix_mask = radix_mask_upper ^ radix_mask_bottom; return key & radix_mask; } -template -constexpr bool is_extended_fp - = std::is_same_v || std::is_same_v - || std::is_same_v || std::is_same_v - || std::is_same_v; - template clauses can be removed once - // libhipcxx is a hard depedency and test types half_t - // and bfloat_t are removed. - _HIPCUB_STD::is_floating_point_v || is_extended_fp, - int> + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + std::enable_if_t::CATEGORY == hipcub::FLOATING_POINT, int> = 0> -auto to_bits(const Key key) +HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP auto to_bits(const Key key) { using unsigned_bits_type = typename hipcub::NumericTraits::UnsignedBits; - static_assert(sizeof(unsigned_bits_type) == sizeof(Key)); unsigned_bits_type bit_key; - std::memcpy(&bit_key, &key, sizeof(unsigned_bits_type)); + memcpy(&bit_key, &key, sizeof(Key)); // Remove signed zero, this case is supposed to be treated the same as // unsigned zero in hipcub sorting algorithms. - constexpr unsigned_bits_type minus_zero = unsigned_bits_type{1} - << (8 * sizeof(unsigned_bits_type) - 1); - + constexpr unsigned_bits_type minus_zero = unsigned_bits_type{1} << (8 * sizeof(Key) - 1); // Positive and negative zero should compare the same. if(bit_key == minus_zero) { @@ -119,7 +96,7 @@ auto to_bits(const Key key) template>::value, int> = 0> + std::enable_if_t::value, int> = 0> auto to_bits(const Key& key) { using inner_t = typename inner_type::type; @@ -131,16 +108,19 @@ auto to_bits(const Key& key) uint32_t, std::conditional_t>>; - auto bit_key_upper = static_cast(to_bits<0, sizeof(inner_t) * 8>(key.x)); - auto bit_key_lower = static_cast(to_bits<0, sizeof(inner_t) * 8>(key.y)); + auto bit_key_upper = static_cast(to_bits<0, sizeof(key.x) * 8>(key.x)); + auto bit_key_lower = static_cast(to_bits<0, sizeof(key.y) * 8>(key.y)); // Flip sign bit to properly order signed types - if(std::is_signed::value) - { - constexpr auto sign_bit = static_cast(1) << (sizeof(inner_t) * 8 - 1); - bit_key_upper ^= sign_bit; - bit_key_lower ^= sign_bit; - } + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + if(::hipcub::NumericTraits::CATEGORY == hipcub::SIGNED_INTEGER) + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + { + constexpr auto sign_bit = static_cast(1) + << (sizeof(inner_t) * 8 - 1); + bit_key_upper ^= sign_bit; + bit_key_lower ^= sign_bit; + } // Create the result containing both parts const auto bit_key diff --git a/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp b/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp index dce0155b29cb..8a85a5f4dd11 100644 --- a/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp +++ b/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023-2026 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -73,7 +73,7 @@ struct ExtendedFloatBoolOp }; /** - * \brief ExtendedFloatBinOp general functor - Because test_utils::plus{}, Difference(), Division(), + * \brief ExtendedFloatBinOp general functor - Because hipcub::Sum(), Difference(), Division(), * Max() and Min() don't work with input types , * and * and . @@ -155,13 +155,13 @@ struct ArgMax { template - || std::is_same_v, + std::enable_if_t::value + || std::is_same::value, bool> = true> - HIPCUB_HOST_DEVICE __forceinline__ - hipcub::KeyValuePair operator()(const hipcub::KeyValuePair& a, - const hipcub::KeyValuePair& b) const + HIPCUB_HOST_DEVICE __forceinline__ hipcub::KeyValuePair + operator()(const hipcub::KeyValuePair& a, + const hipcub::KeyValuePair& b) const { const hipcub::KeyValuePair native_a(a.key, a.value); const hipcub::KeyValuePair native_b(b.key, b.value); @@ -179,13 +179,13 @@ struct ArgMin { template - || std::is_same_v, + std::enable_if_t::value + || std::is_same::value, bool> = true> - HIPCUB_HOST_DEVICE __forceinline__ - hipcub::KeyValuePair operator()(const hipcub::KeyValuePair& a, - const hipcub::KeyValuePair& b) const + HIPCUB_HOST_DEVICE __forceinline__ hipcub::KeyValuePair + operator()(const hipcub::KeyValuePair& a, + const hipcub::KeyValuePair& b) const { const hipcub::KeyValuePair native_a(a.key, a.value); const hipcub::KeyValuePair native_b(b.key, b.value); @@ -279,50 +279,50 @@ struct AlgebraicSelector template struct MaxSelector { - using type = test_utils::maximum; + using type = hipcub::Max; }; template struct MaxSelector, test_utils::custom_test_type> { - using type = CustomTestOp; + using type = CustomTestOp; }; template struct MaxSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; template struct MaxSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; // Min functor selector. template struct MinSelector { - using type = test_utils::minimum; + using type = hipcub::Min; }; template struct MinSelector, test_utils::custom_test_type> { - using type = CustomTestOp; + using type = CustomTestOp; }; template struct MinSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; template struct MinSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; // ArgMax functor selector diff --git a/projects/hipcub/toolchain-windows.cmake b/projects/hipcub/toolchain-windows.cmake index 313ab047ba7b..6b688314c935 100644 --- a/projects/hipcub/toolchain-windows.cmake +++ b/projects/hipcub/toolchain-windows.cmake @@ -31,11 +31,4 @@ if (DEFINED ENV{VCPKG_PATH}) else() set(VCPKG_PATH "C:/github/vcpkg") endif() - -# Force static libraries on Windows -set(VCPKG_TARGET_TRIPLET "x64-windows-static") - -# Force static MSVC runtime (/MT) to match vcpkg static triplet -set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") - include("${VCPKG_PATH}/scripts/buildsystems/vcpkg.cmake") diff --git a/projects/rocprim/CHANGELOG.md b/projects/rocprim/CHANGELOG.md index 78c274777679..681930d039d6 100644 --- a/projects/rocprim/CHANGELOG.md +++ b/projects/rocprim/CHANGELOG.md @@ -9,15 +9,6 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec * Added a parallel `device_topk`, which finds the largest/smallest K elements from an input array of keys. * Added a parallel `device_segmented_topk`, which finds the largest/smallest K elements from segmented groups. * `device_topk` and `device_segmented_topk` are controlled by cmake flag `ROCPRIM_ENABLE_TOPK`. Passing `-DROCPRIM_ENABLE_TOPK=ON` to enable these features -* Added C++ 17 style type_traits utilities - * is_floating_point_v - * is_integral_v - * is_arithmetic_v - * is_fundamental_v - * is_unsigned_v - * is_signed_v - * is_scalar_v - * is_compound_v ### Changed diff --git a/projects/rocprim/rmake.py b/projects/rocprim/rmake.py index c1f25a4c2ba8..ae6274876787 100644 --- a/projects/rocprim/rmake.py +++ b/projects/rocprim/rmake.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -""" Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. +""" Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. Manage build and installation""" import re diff --git a/projects/rocprim/rocprim/include/rocprim/type_traits.hpp b/projects/rocprim/rocprim/include/rocprim/type_traits.hpp index f8e5cffb2b8e..10e602c869a2 100644 --- a/projects/rocprim/rocprim/include/rocprim/type_traits.hpp +++ b/projects/rocprim/rocprim/include/rocprim/type_traits.hpp @@ -1425,34 +1425,6 @@ template struct is_compound : std::integral_constant().is_compound()> {}; -#ifndef DOXYGEN_DOCUMENTATION_BUILD - -template -constexpr bool is_floating_point_v = is_floating_point::value; - -template -constexpr bool is_integral_v = is_integral::value; - -template -constexpr bool is_arithmetic_v = is_arithmetic::value; - -template -constexpr bool is_fundamental_v = is_fundamental::value; - -template -constexpr bool is_unsigned_v = is_unsigned::value; - -template -constexpr bool is_signed_v = is_unsigned::value; - -template -constexpr bool is_scalar_v = is_scalar::value; - -template -constexpr bool is_compound_v = is_compound::value; - -#endif - static_assert(::rocprim::traits::radix_key_codec::radix_key_fundamental::value, "'int' should be fundamental"); static_assert(!::rocprim::traits::radix_key_codec::radix_key_fundamental::value, @@ -1464,13 +1436,6 @@ static_assert(::rocprim::traits::radix_key_codec::radix_key_fundamental::value, "'rocprim::int128_t*' should not be fundamental"); -static_assert(::rocprim::is_floating_point_v<__half>, "__half should be a floating point type"); -static_assert(::rocprim::is_floating_point_v, "bfloat16 should be a floating point type"); -static_assert(::rocprim::is_integral_v<::rocprim::int128_t>, - "::rocprim::int128_t should be a integral type"); -static_assert(::rocprim::is_integral_v<::rocprim::uint128_t>, - "::rocprim::uint128_t should be a integral type"); - END_ROCPRIM_NAMESPACE #endif diff --git a/projects/rocthrust/test/test_scan.cpp b/projects/rocthrust/test/test_scan.cpp index a86a9fd2fcaf..dc15c9f7ae3b 100644 --- a/projects/rocthrust/test/test_scan.cpp +++ b/projects/rocthrust/test/test_scan.cpp @@ -568,11 +568,6 @@ TEST(ScanTests, TestScanWithLargeTypes) #ifdef ADDRESS_SANITIZER_BUILD GTEST_SKIP() << "Skipping test due to memory constraints in address sanitizer build."; #endif - - // Temporarily disable this test on gfx115x on Windows until we can determine the root cause. - // TODO: remove this after the root cause has been found and fixed properly. - if (temp_skip::should_skip()) - GTEST_SKIP() << "Skipping test on gfx1151 Windows systems due to known issues."; SCOPED_TRACE(testing::Message() << "with device_id= " << test::set_device_from_ctest()); diff --git a/projects/rocthrust/test/test_utils.hpp b/projects/rocthrust/test/test_utils.hpp index 06cca8d82cc0..a0a7609c82b8 100644 --- a/projects/rocthrust/test/test_utils.hpp +++ b/projects/rocthrust/test/test_utils.hpp @@ -52,11 +52,6 @@ #include #include -#if defined(_WIN32) && defined(_THRUST_USE_ROCPRIM) -#include -#include -#endif - // HIP API #if THRUST_HAS_HIP_COMPILER() # include @@ -82,32 +77,6 @@ #endif // THRUST_DEVICE_COMPILER == THRUST_DEVICE_COMPILER_HIP -// Temporarily disable some tests on gfx115x on Windows until we can determine the root cause of the failures. - // TODO: remove this after the root cause has been found and fixed properly. -namespace temp_skip -{ - bool should_skip() - { -#if defined(_WIN32) && defined(_THRUST_USE_ROCPRIM) - rocprim::detail::target_arch arch = rocprim::detail::target_arch::unknown; - if (rocprim::detail::host_target_arch(hipStreamDefault, arch) != HIP_SUCCESS) - { - std::cerr << "Warning: unable to fetch target architecture for disablement check." << std::endl; - } - - const std::set disabled_arches = { - rocprim::detail::target_arch::gfx1150, - rocprim::detail::target_arch::gfx1151, - rocprim::detail::target_arch::gfx1152, - rocprim::detail::target_arch::gfx1153 - }; - - if (disabled_arches.find(arch) != disabled_arches.end()) - return true; -#endif - return false; - } -}; namespace test { diff --git a/projects/rocthrust/test/test_zip_iterator_sort_by_key.cpp b/projects/rocthrust/test/test_zip_iterator_sort_by_key.cpp index 498083353535..9e8e8ad6a31d 100644 --- a/projects/rocthrust/test/test_zip_iterator_sort_by_key.cpp +++ b/projects/rocthrust/test/test_zip_iterator_sort_by_key.cpp @@ -28,11 +28,6 @@ TESTS_DEFINE(ZipIteratorStableSortByKeyTests, TestParams); TYPED_TEST(ZipIteratorStableSortByKeyTests, TestZipIteratorStableSortByKey) { - // Temporarily disable this test on gfx115x on Windows until we can determine the root cause. - // TODO: remove this after the root cause has been found and fixed properly. - if (temp_skip::should_skip()) - GTEST_SKIP() << "Skipping test on gfx1151 Windows systems due to known issues."; - using T = typename TestFixture::input_type; using namespace thrust;