diff --git a/.github/workflows/wheels-build-stage.yaml b/.github/workflows/wheels-build-stage.yaml index d6325b56c5e1..72c214afcb1f 100644 --- a/.github/workflows/wheels-build-stage.yaml +++ b/.github/workflows/wheels-build-stage.yaml @@ -105,31 +105,15 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} with: log-file: ${{ env.RAPIDS_ARTIFACTS_DIR }}/sccache.log - - name: Restore Cython cache - id: cython-cache-restore - if: inputs.stage == 'python' - uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ${{ github.workspace }}/.cache/cython - key: cython-v1-${{ runner.os }}-${{ runner.arch }}-cuda${{ matrix.CUDA_VER }}-py${{ matrix.PY_VER }}-${{ hashFiles('dependencies.yaml', 'python/**/*.pyx', 'python/**/*.pxd', 'python/**/CMakeLists.txt', 'python/**/pyproject.toml') }} - restore-keys: | - cython-v1-${{ runner.os }}-${{ runner.arch }}-cuda${{ matrix.CUDA_VER }}-py${{ matrix.PY_VER }}- - name: Build staged wheels id: build-wheels run: | ulimit -n "$(ulimit -Hn)" source "${INPUTS_SCRIPT}" env: - CYTHON_CACHE_DIR: ${{ github.workspace }}/.cache/cython GH_TOKEN: ${{ github.token }} INPUTS_SCRIPT: ${{ inputs.script }} shell: bash -leo pipefail {0} - - name: Save Cython cache - if: inputs.stage == 'python' && steps.cython-cache-restore.outputs.cache-hit != 'true' - uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ${{ github.workspace }}/.cache/cython - key: ${{ steps.cython-cache-restore.outputs.cache-primary-key }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: inputs.stage == 'cpp' with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb8d626cb3df..24d0f3761ba5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: hooks: - id: mypy additional_dependencies: [ - "polars>=1.35,<1.43", + "polars>=1.35,<1.45", "numpy>=1.26", "pyarrow-stubs>=19.0", "pyarrow>=19.0.0,<24.0.0", # https://github.com/NVIDIA/cudf/issues/22229 diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index bc243af5d33d..84184eddc00a 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -78,7 +78,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.1.0 - pandoc -- polars>=1.35,<1.43 +- polars>=1.35,<1.45 - pre-commit - psutil - pyarrow>=19.0.0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 5dd12b095574..09151dd6b9d0 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -78,7 +78,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.1.0 - pandoc -- polars>=1.35,<1.43 +- polars>=1.35,<1.45 - pre-commit - psutil - pyarrow>=19.0.0 diff --git a/conda/environments/all_cuda-133_arch-aarch64.yaml b/conda/environments/all_cuda-133_arch-aarch64.yaml index e2ad3535b15c..5448a29d2136 100644 --- a/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -78,7 +78,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.1.0 - pandoc -- polars>=1.35,<1.43 +- polars>=1.35,<1.45 - pre-commit - psutil - pyarrow>=19.0.0 diff --git a/conda/environments/all_cuda-133_arch-x86_64.yaml b/conda/environments/all_cuda-133_arch-x86_64.yaml index 567bf8b161c6..4e53613ba77f 100644 --- a/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -78,7 +78,7 @@ dependencies: - packaging - pandas>=3.0.0,<3.1.0 - pandoc -- polars>=1.35,<1.43 +- polars>=1.35,<1.45 - pre-commit - psutil - pyarrow>=19.0.0 diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index a59fedabd19a..15da8358c8bf 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -39,7 +39,7 @@ requirements: - rapidsmpf =${{ minor_version }} - kvikio =${{ minor_version }} - cudf-streaming =${{ version }} - - polars>=1.35,<1.43 + - polars>=1.35,<1.45 - packaging - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "12" diff --git a/cpp/include/cudf/io/experimental/hybrid_scan_multifile.hpp b/cpp/include/cudf/io/experimental/hybrid_scan_multifile.hpp index c75fa3d186d3..23e4b5b44129 100644 --- a/cpp/include/cudf/io/experimental/hybrid_scan_multifile.hpp +++ b/cpp/include/cudf/io/experimental/hybrid_scan_multifile.hpp @@ -61,6 +61,8 @@ class hybrid_scan_multifile { /** * @brief Constructor for the multi-file experimental Parquet reader * + * @throws std::invalid_argument if no sources are provided + * * @param footer_bytes Host span of Parquet file footer byte spans, one per source * @param options Parquet reader options */ @@ -70,12 +72,25 @@ class hybrid_scan_multifile { /** * @brief Constructor for the multi-file experimental Parquet reader * + * @throws std::invalid_argument if no sources are provided + * * @param parquet_metadata Host span of pre-populated Parquet file metadata, one per source * @param options Parquet reader options */ explicit hybrid_scan_multifile(cudf::host_span parquet_metadata, parquet_reader_options const& options); + /** + * @brief Constructor that takes ownership of pre-populated Parquet file metadata + * + * @throws std::invalid_argument if no sources are provided + * + * @param parquet_metadata Pre-populated Parquet file metadata, one per source + * @param options Parquet reader options + */ + explicit hybrid_scan_multifile(std::vector&& parquet_metadata, + parquet_reader_options const& options); + /** * @brief Destructor for the multi-file experimental Parquet reader */ diff --git a/cpp/include/cudf/utilities/span.hpp b/cpp/include/cudf/utilities/span.hpp index 22a0dff9e295..0fcacecac12f 100644 --- a/cpp/include/cudf/utilities/span.hpp +++ b/cpp/include/cudf/utilities/span.hpp @@ -107,26 +107,28 @@ struct host_span { /// Constructor from container /// @param in The container to construct the span from - template ::value && - std::is_convertible_v< - std::remove_pointer_t().data()))> (*)[], - T (*)[]>>* = nullptr> // NOLINT + template < + typename C, + // Only supported containers of types convertible to T + std::enable_if_t< + is_host_span_supported_container::value && + std::is_convertible_v().data()))> (*)[], // NOLINT(modernize-type-traits) + T (*)[]>>* = nullptr> // NOLINT constexpr host_span(C& in) : _span{thrust::raw_pointer_cast(in.data()), in.size()} { } /// Constructor from const container /// @param in The container to construct the span from - template ::value && - std::is_convertible_v< - std::remove_pointer_t().data()))> (*)[], - T (*)[]>>* = nullptr> // NOLINT + template < + typename C, + // Only supported containers of types convertible to T + std::enable_if_t< + is_host_span_supported_container::value && + std::is_convertible_v().data()))> (*)[], // NOLINT(modernize-type-traits) + T (*)[]>>* = nullptr> // NOLINT constexpr host_span(C const& in) : _span{thrust::raw_pointer_cast(in.data()), in.size()} { } diff --git a/cpp/src/io/parquet/compact_protocol_reader.hpp b/cpp/src/io/parquet/compact_protocol_reader.hpp index 00634d3065fb..a4c59807111e 100644 --- a/cpp/src/io/parquet/compact_protocol_reader.hpp +++ b/cpp/src/io/parquet/compact_protocol_reader.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2018-2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -8,12 +8,17 @@ #include "parquet_common.hpp" #include +#include #include #include #include +#include #include +#include +#include +#include #include namespace CUDF_EXPORT cudf { @@ -36,10 +41,20 @@ class CompactProtocolReader { explicit CompactProtocolReader(uint8_t const* base = nullptr, size_t len = 0) { init(base, len); } void init(uint8_t const* base, size_t len) { + // A null base is valid only for an empty buffer; a positive length would then have no backing + // storage. This keeps every later pointer op defined (the empty state has all-null pointers). + CUDF_EXPECTS(base != nullptr || len == 0, + "CompactProtocolReader requires a non-null buffer when length is non-zero", + std::invalid_argument); m_base = m_cur = base; - m_end = base + len; + // Guard against `nullptr + len` (undefined) so a zero-length buffer stays fully defined. + m_end = base != nullptr ? base + len : base; + } + [[nodiscard]] ptrdiff_t bytecount() const noexcept + { + // Avoid `nullptr - nullptr` on a null-base reader; it has consumed nothing. + return m_base != nullptr ? m_cur - m_base : 0; } - [[nodiscard]] ptrdiff_t bytecount() const noexcept { return m_cur - m_base; } unsigned int getb() noexcept { return (m_cur < m_end) ? *m_cur++ : 0; } void skip_bytes(size_t bytecnt) noexcept { @@ -47,13 +62,20 @@ class CompactProtocolReader { m_cur += bytecnt; } - // returns a varint encoded integer - template - T get_varint() noexcept + // Returns a varint-encoded integer. `T` is constrained to unsigned so `numeric_limits::digits` + // is the full value width; a signed `T` would drop the sign bit and misplace the overflow bound. + template + T get_varint() { T v = 0; for (uint32_t l = 0;; l += 7) { - T c = getb(); + T const c = getb(); + // The byte's value, shifted into place, must fit in `T`; `l < digits` also keeps `max() >> l` + // itself in range. Comparing the raw byte, not the masked payload, is intentional: it also + // rejects a continuation byte whose successor group could not fit. + CUDF_EXPECTS(l < std::numeric_limits::digits && c <= (std::numeric_limits::max() >> l), + "Parquet varint exceeds the width of its target type", + std::overflow_error); v |= (c & 0x7f) << l; if (c < 0x80) { break; } } @@ -62,7 +84,7 @@ class CompactProtocolReader { // returns a zigzag encoded signed integer template - T get_zigzag() noexcept + T get_zigzag() { using U = std::make_unsigned_t; U const u = get_varint(); @@ -70,14 +92,14 @@ class CompactProtocolReader { } // thrift spec says to use zigzag i32 for i16 types - int32_t get_i16() noexcept { return get_zigzag(); } - int32_t get_i32() noexcept { return get_zigzag(); } - int64_t get_i64() noexcept { return get_zigzag(); } + int32_t get_i16() { return get_zigzag(); } + int32_t get_i32() { return get_zigzag(); } + int64_t get_i64() { return get_zigzag(); } - uint32_t get_u32() noexcept { return get_varint(); } - uint64_t get_u64() noexcept { return get_varint(); } + uint32_t get_u32() { return get_varint(); } + uint64_t get_u64() { return get_varint(); } - [[nodiscard]] std::pair get_listh() noexcept + [[nodiscard]] std::pair get_listh() { uint32_t const c = getb(); uint32_t sz = c >> 4; diff --git a/cpp/src/io/parquet/experimental/hybrid_scan.cpp b/cpp/src/io/parquet/experimental/hybrid_scan.cpp index c133ed3130ea..bd6a159b6ea2 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan.cpp @@ -18,7 +18,7 @@ hybrid_scan_metadata::hybrid_scan_metadata(cudf::host_span footer : _metadata{std::make_shared( std::vector>{footer_bytes}, options.is_enabled_use_arrow_schema(), - options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas())} + options.is_enabled_allow_mismatched_pq_schemas())} { } @@ -27,7 +27,7 @@ hybrid_scan_metadata::hybrid_scan_metadata(FileMetaData const& parquet_metadata, : _metadata{std::make_shared( std::vector{parquet_metadata}, options.is_enabled_use_arrow_schema(), - options.get_column_names().has_value() and options.is_enabled_allow_mismatched_pq_schemas())} + options.is_enabled_allow_mismatched_pq_schemas())} { } diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp index faba65f4f33a..3502b730abbe 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.cpp @@ -36,22 +36,6 @@ using text::byte_range_info; namespace { -// Construct a vector of FileMetaData from the input footer bytes -[[nodiscard]] std::vector parquet_metadatas_from_footer_bytes( - cudf::host_span const> footer_bytes) -{ - std::vector parquet_metadatas; - parquet_metadatas.reserve(footer_bytes.size()); - std::transform(footer_bytes.begin(), - footer_bytes.end(), - std::back_inserter(parquet_metadatas), - [](auto const& footer_bytes) { - metadata parsed_metadata{footer_bytes}; - return FileMetaData{std::move(parsed_metadata)}; - }); - return parquet_metadatas; -} - // Construct a vector of all row group indices from the input vectors [[nodiscard]] auto all_row_group_indices( std::span const> row_group_indices) @@ -77,8 +61,10 @@ namespace { } // Compute the page index (column index and/or offset index) byte range -[[nodiscard]] byte_range_info page_index_byte_range(FileMetaData const& file_metadata) +[[nodiscard]] byte_range_info page_index_byte_range(parquet::detail::metadata const& file_metadata) { + if (file_metadata.is_page_index_setup()) { return {}; } + auto const& row_groups = file_metadata.row_groups; if (row_groups.empty() or row_groups.front().columns.empty()) { return {}; } @@ -132,25 +118,31 @@ aggregate_reader_metadata::aggregate_reader_metadata( cudf::host_span const> footer_bytes, bool use_arrow_schema, bool has_cols_from_mismatched_srcs) - : aggregate_reader_metadata_base(parquet_metadatas_from_footer_bytes(footer_bytes), - use_arrow_schema, - has_cols_from_mismatched_srcs) + : aggregate_reader_metadata( + parquet::detail::parallel_construct_metadatas( + footer_bytes, [](auto const& bytes) { return FileMetaData{metadata{bytes}}; }), + use_arrow_schema, + has_cols_from_mismatched_srcs) { - CUDF_EXPECTS( - not footer_bytes.empty(), "At least one source must be provided", std::invalid_argument); } aggregate_reader_metadata::aggregate_reader_metadata( cudf::host_span parquet_metadatas, bool use_arrow_schema, bool has_cols_from_mismatched_srcs) - : aggregate_reader_metadata_base( + : aggregate_reader_metadata( std::vector{parquet_metadatas.begin(), parquet_metadatas.end()}, use_arrow_schema, has_cols_from_mismatched_srcs) { - CUDF_EXPECTS( - not parquet_metadatas.empty(), "At least one source must be provided", std::invalid_argument); +} + +aggregate_reader_metadata::aggregate_reader_metadata(std::vector&& parquet_metadatas, + bool use_arrow_schema, + bool has_cols_from_mismatched_srcs) + : aggregate_reader_metadata_base( + std::move(parquet_metadatas), use_arrow_schema, has_cols_from_mismatched_srcs) +{ } std::vector aggregate_reader_metadata::page_index_byte_ranges() const diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp index e0a3746ecf5b..075d85660d97 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_helpers.hpp @@ -81,6 +81,8 @@ class aggregate_reader_metadata : public aggregate_reader_metadata_base { /** * @brief Constructor for aggregate_reader_metadata * + * @throws std::invalid_argument if no sources are provided + * * @param footer_bytes Host span of Parquet file footer buffer bytes, one per source * @param use_arrow_schema Whether to use Arrow schema * @param has_cols_from_mismatched_srcs Whether to have columns from mismatched sources @@ -92,6 +94,8 @@ class aggregate_reader_metadata : public aggregate_reader_metadata_base { /** * @brief Constructor for aggregate_reader_metadata * + * @throws std::invalid_argument if no sources are provided + * * @param parquet_metadatas Host span of pre-populated Parquet file metadata, one per source * @param use_arrow_schema Whether to use Arrow schema * @param has_cols_from_mismatched_srcs Whether to have columns from mismatched sources @@ -100,6 +104,19 @@ class aggregate_reader_metadata : public aggregate_reader_metadata_base { bool use_arrow_schema, bool has_cols_from_mismatched_srcs); + /** + * @brief Constructor that takes ownership of pre-populated Parquet file metadata + * + * @throws std::invalid_argument if no sources are provided + * + * @param parquet_metadatas Pre-populated Parquet file metadata, one per source + * @param use_arrow_schema Whether to use Arrow schema + * @param has_cols_from_mismatched_srcs Whether to have columns from mismatched sources + */ + aggregate_reader_metadata(std::vector&& parquet_metadatas, + bool use_arrow_schema, + bool has_cols_from_mismatched_srcs); + aggregate_reader_metadata(aggregate_reader_metadata const&) = delete; aggregate_reader_metadata& operator=(aggregate_reader_metadata const&) = delete; aggregate_reader_metadata(aggregate_reader_metadata&&) = default; diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp b/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp index 61e159f9f837..59cd164d3e6d 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_impl.cpp @@ -180,8 +180,10 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl( cudf::host_span const> footer_bytes, parquet_reader_options const& options) { - _metadata = std::make_shared( - footer_bytes, options.is_enabled_use_arrow_schema(), has_cols_from_mismatched_sources(options)); + _metadata = + std::make_shared(footer_bytes, + options.is_enabled_use_arrow_schema(), + options.is_enabled_allow_mismatched_pq_schemas()); _extended_metadata = static_cast(_metadata.get()); } @@ -192,7 +194,17 @@ hybrid_scan_reader_impl::hybrid_scan_reader_impl( _metadata = std::make_shared(parquet_metadatas, options.is_enabled_use_arrow_schema(), - has_cols_from_mismatched_sources(options)); + options.is_enabled_allow_mismatched_pq_schemas()); + _extended_metadata = static_cast(_metadata.get()); +} + +hybrid_scan_reader_impl::hybrid_scan_reader_impl(std::vector&& parquet_metadatas, + parquet_reader_options const& options) +{ + _metadata = + std::make_shared(std::move(parquet_metadatas), + options.is_enabled_use_arrow_schema(), + options.is_enabled_allow_mismatched_pq_schemas()); _extended_metadata = static_cast(_metadata.get()); } @@ -285,12 +297,13 @@ void hybrid_scan_reader_impl::select_columns(read_columns_mode read_columns_mode // Save original output-buffer schema for reuse across materialization passes. _original_output_buffers_template = make_empty_like_column_buffers(_output_buffers); - // Initialize mutable output-buffer template for this materialization pass. - reset_output_buffers_template(); + // Initialize mutable output buffers for this materialization pass. + reset_output_buffers(); } -void hybrid_scan_reader_impl::reset_output_buffers_template() +void hybrid_scan_reader_impl::reset_output_buffers() { + _output_buffers = make_empty_like_column_buffers(_original_output_buffers_template); _output_buffers_template = make_empty_like_column_buffers(_original_output_buffers_template); } @@ -336,7 +349,7 @@ void hybrid_scan_reader_impl::prepare_materialization(read_columns_mode read_col reset_internal_state(); initialize_options(options, num_sources, stream, mr); select_columns(read_columns_mode, options); - reset_output_buffers_template(); + reset_output_buffers(); } std::vector> diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp index bbdacd818a4e..c3feb436b82e 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_impl.hpp @@ -58,6 +58,15 @@ class hybrid_scan_reader_impl : public parquet::detail::reader_impl { explicit hybrid_scan_reader_impl(cudf::host_span parquet_metadatas, parquet_reader_options const& options); + /** + * @brief Constructor that takes ownership of pre-populated Parquet file metadata + * + * @param parquet_metadatas Pre-populated Parquet file metadata, one per source + * @param options Parquet reader options + */ + explicit hybrid_scan_reader_impl(std::vector&& parquet_metadatas, + parquet_reader_options const& options); + /** * @brief Constructor that takes shared ownership of pre-parsed Parquet metadata * @@ -393,9 +402,9 @@ class hybrid_scan_reader_impl : public parquet::detail::reader_impl { void mark_buffers_nullable_for_pruned_pages(); /** - * @brief Initialize the mutable output-buffer template for this materialization + * @brief Reset the output buffers and their template from the original selected-columns schema */ - void reset_output_buffers_template(); + void reset_output_buffers(); /** * @brief Select the columns to be read based on the read mode diff --git a/cpp/src/io/parquet/experimental/hybrid_scan_multifile.cpp b/cpp/src/io/parquet/experimental/hybrid_scan_multifile.cpp index 259435401aab..493c86a8f3b2 100644 --- a/cpp/src/io/parquet/experimental/hybrid_scan_multifile.cpp +++ b/cpp/src/io/parquet/experimental/hybrid_scan_multifile.cpp @@ -26,6 +26,12 @@ hybrid_scan_multifile::hybrid_scan_multifile(cudf::host_span { } +hybrid_scan_multifile::hybrid_scan_multifile(std::vector&& parquet_metadata, + parquet_reader_options const& options) + : _impl{std::make_unique(std::move(parquet_metadata), options)} +{ +} + hybrid_scan_multifile::~hybrid_scan_multifile() = default; std::vector hybrid_scan_multifile::parquet_metadatas() const diff --git a/cpp/src/io/parquet/reader_impl.cpp b/cpp/src/io/parquet/reader_impl.cpp index d263ac46d9a2..c2799f781e7c 100644 --- a/cpp/src/io/parquet/reader_impl.cpp +++ b/cpp/src/io/parquet/reader_impl.cpp @@ -624,7 +624,8 @@ void reader_impl::populate_metadata(table_metadata& out_metadata) auto const& schema = _metadata->get_schema(_output_column_schemas[i]); out_metadata.schema_info[i].name = schema.name; out_metadata.schema_info[i].is_nullable = - schema.repetition_type != FieldRepetitionType::REQUIRED; + schema.repetition_type != FieldRepetitionType::REQUIRED or + _metadata->is_nullable_across_sources(_output_column_schemas[i]); } // Return user metadata diff --git a/cpp/src/io/parquet/reader_impl.hpp b/cpp/src/io/parquet/reader_impl.hpp index 6fbd4789c6dd..137d43c0316f 100644 --- a/cpp/src/io/parquet/reader_impl.hpp +++ b/cpp/src/io/parquet/reader_impl.hpp @@ -413,19 +413,6 @@ class reader_impl { _file_itm_data._current_input_pass < _file_itm_data.num_passes(); } - /** - * @brief Check if the user has specified columns from mismatched sources - * - * @param options Reader options - * @return True if the user has specified columns from mismatched sources - */ - [[nodiscard]] bool has_cols_from_mismatched_sources(parquet_reader_options const& options) const - { - return (options.get_column_names().has_value() or - options.get_column_field_ids().has_value()) and - options.is_enabled_allow_mismatched_pq_schemas(); - } - /** * @brief Effective `ignore_missing_columns` policy for column selection * @@ -440,6 +427,20 @@ class reader_impl { not(has_cols_from_mismatched_sources(options) and _metadata->get_num_sources() > 1); } + private: + /** + * @brief Check if the user has specified columns from mismatched sources + * + * @param options Reader options + * @return True if the user has specified columns from mismatched sources + */ + [[nodiscard]] bool has_cols_from_mismatched_sources(parquet_reader_options const& options) const + { + return (options.get_column_names().has_value() or + options.get_column_field_ids().has_value()) and + options.is_enabled_allow_mismatched_pq_schemas(); + } + protected: /** * @brief Check if the user has specified custom row bounds diff --git a/cpp/src/io/parquet/reader_impl_chunking.cu b/cpp/src/io/parquet/reader_impl_chunking.cu index 1cb95bd35a97..f728ebfc00cc 100644 --- a/cpp/src/io/parquet/reader_impl_chunking.cu +++ b/cpp/src/io/parquet/reader_impl_chunking.cu @@ -19,6 +19,7 @@ #include #include +#include namespace cudf::io::parquet::detail { @@ -419,29 +420,32 @@ void reader_impl::create_global_chunk_info() auto const num_chunks = row_groups_info.size() * num_input_columns; // Mapping of input column to page index column - std::vector column_mapping; - - if (_has_offset_index and not row_groups_info.empty()) { - // use first row group to define mappings (assumes same schema for each file) - auto const& rg = row_groups_info[0]; - auto const& columns = _metadata->get_row_group(rg.index, rg.source_index).columns; - column_mapping.resize(num_input_columns); - std::transform( - _input_columns.begin(), _input_columns.end(), column_mapping.begin(), [&](auto const& col) { - // translate schema_idx into something we can use for the page indexes - if (auto it = std::find_if(columns.begin(), - columns.end(), - [&](auto const& col_chunk) { - return col_chunk.schema_idx == - _metadata->map_schema_index(col.schema_idx, - rg.source_index); - }); - it != columns.end()) { - return std::distance(columns.begin(), it); - } - CUDF_FAIL("cannot find column mapping"); - }); - } + auto column_mappings = std::unordered_map>{}; + + auto const column_mapping_for_source = [&](auto const& rg) -> std::vector const& { + auto const [iter, inserted] = column_mappings.try_emplace(rg.source_index); + if (inserted) { + auto const& columns = _metadata->get_row_group(rg.index, rg.source_index).columns; + auto& mapping = iter->second; + mapping.resize(num_input_columns); + std::transform( + _input_columns.begin(), _input_columns.end(), mapping.begin(), [&](auto const& col) { + // translate schema_idx into something we can use for the page indexes + if (auto it = std::find_if(columns.begin(), + columns.end(), + [&](auto const& col_chunk) { + return col_chunk.schema_idx == + _metadata->map_schema_index(col.schema_idx, + rg.source_index); + }); + it != columns.end()) { + return static_cast(std::distance(columns.begin(), it)); + } + CUDF_FAIL("cannot find column mapping"); + }); + } + return iter->second; + }; // Initialize column chunk information auto remaining_rows = num_rows; @@ -454,6 +458,8 @@ void reader_impl::create_global_chunk_info() auto row_group_rows = std::min(remaining_rows + adjusted_row_group_rows, row_group.num_rows); + auto const* const column_mapping = _has_offset_index ? &column_mapping_for_source(rg) : nullptr; + // generate ColumnChunkDesc objects for everything to be decoded (all input columns) for (size_t i = 0; i < num_input_columns; ++i) { auto col = _input_columns[i]; @@ -479,7 +485,7 @@ void reader_impl::create_global_chunk_info() // grab the column_chunk_info for each chunk (if it exists) column_chunk_info const* const chunk_info = - _has_offset_index ? &rg.column_chunks.value()[column_mapping[i]] : nullptr; + _has_offset_index ? &rg.column_chunks.value()[(*column_mapping)[i]] : nullptr; chunks.emplace_back(col_meta.total_compressed_size, nullptr, diff --git a/cpp/src/io/parquet/reader_impl_helpers.cpp b/cpp/src/io/parquet/reader_impl_helpers.cpp index f229a6120721..b93da3ffc30d 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.cpp +++ b/cpp/src/io/parquet/reader_impl_helpers.cpp @@ -510,7 +510,17 @@ void metadata::sanitize_schema() process(0); } -metadata::metadata(FileMetaData&& other) : FileMetaData(std::move(other)) {} +metadata::metadata(FileMetaData&& other) : FileMetaData(std::move(other)) +{ + // Since page index is set up for all or no row groups, just check if any column chunk has it set. + // Update this check if this behavior changes in the future. + is_page_index_setup_ = + std::any_of(row_groups.cbegin(), row_groups.cend(), [](auto const& row_group) { + return std::any_of(row_group.columns.cbegin(), row_group.columns.cend(), [](auto const& col) { + return col.column_index.has_value() or col.offset_index.has_value(); + }); + }); +} metadata::metadata(datasource* source, bool read_page_indexes) { @@ -550,6 +560,8 @@ metadata::metadata(datasource* source, bool read_page_indexes) void metadata::setup_page_index(cudf::host_span page_index_bytes, int64_t min_offset) { + if (is_page_index_setup_) { return; } + CUDF_FUNC_RANGE(); // Flatten all columns into a single vector for easier task distribution @@ -625,6 +637,8 @@ void metadata::setup_page_index(cudf::host_span page_index_bytes, read_column_indexes(cp, col_ref.get()); } } + + is_page_index_setup_ = true; } metadata::~metadata() @@ -645,26 +659,9 @@ metadata::~metadata() std::vector aggregate_reader_metadata::metadatas_from_sources( host_span const> sources, bool read_page_indexes) { - // Avoid using the thread pool for a single source - if (sources.size() == 1) { - std::vector result; - result.emplace_back(sources[0].get(), read_page_indexes); - return result; - } - - std::vector> metadata_ctor_tasks; - metadata_ctor_tasks.reserve(sources.size()); - for (auto const& source : sources) { - metadata_ctor_tasks.emplace_back(cudf::detail::host_worker_pool().submit_task( - [source = source.get(), read_page_indexes] { return metadata{source, read_page_indexes}; })); - } - std::vector metadatas; - metadatas.reserve(sources.size()); - std::transform(metadata_ctor_tasks.begin(), - metadata_ctor_tasks.end(), - std::back_inserter(metadatas), - [](std::future& task) { return std::move(task).get(); }); - return metadatas; + return parallel_construct_metadatas(sources, [read_page_indexes](auto const& source) { + return metadata{source.get(), read_page_indexes}; + }); } std::vector> @@ -901,6 +898,15 @@ bool aggregate_reader_metadata::has_offset_index( return true; } +void aggregate_reader_metadata::propagate_optional_field(int schema_idx, SchemaElement const& src) +{ + if (per_file_metadata.front().schema[schema_idx].repetition_type == + FieldRepetitionType::REQUIRED and + src.repetition_type != FieldRepetitionType::REQUIRED) { + nullable_across_sources.insert(schema_idx); + } +} + void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, bool has_cols_from_mismatched_srcs) { @@ -926,23 +932,18 @@ void aggregate_reader_metadata::initialize_internals(bool use_arrow_schema, } CUDF_EXPECTS(schema == pfm.schema, "All sources must have the same schema"); } - } - // Mark the column schema in the first (default) source as nullable if it is nullable in any of - // the input sources. This avoids recomputing this within build_column() and - // populate_metadata(). - std::for_each( - cuda::counting_iterator{static_cast(1)}, - cuda::counting_iterator{schema.size()}, - [&](auto const schema_idx) { - if (schema[schema_idx].repetition_type == FieldRepetitionType::REQUIRED and - std::any_of( - per_file_metadata.begin() + 1, per_file_metadata.end(), [&](auto const& pfm) { - return pfm.schema[schema_idx].repetition_type != FieldRepetitionType::REQUIRED; - })) { - schema[schema_idx].repetition_type = FieldRepetitionType::OPTIONAL; - } - }); + // Record fields that are nullable in any source other than the first one + std::for_each( + cuda::counting_iterator{static_cast(1)}, + cuda::counting_iterator{schema.size()}, + [&](auto const schema_idx) { + std::for_each( + per_file_metadata.begin() + 1, per_file_metadata.end(), [&](auto const& pfm) { + propagate_optional_field(static_cast(schema_idx), pfm.schema[schema_idx]); + }); + }); + } } // Collect and apply arrow:schema from Parquet's key value metadata section @@ -957,6 +958,10 @@ aggregate_reader_metadata::aggregate_reader_metadata(std::vector&& bool use_arrow_schema, bool has_cols_from_mismatched_srcs) { + CUDF_EXPECTS(not parquet_metadatas.empty(), + "Cannot construct aggregate Parquet reader metadata without source metadata", + std::invalid_argument); + per_file_metadata.reserve(parquet_metadatas.size()); std::transform(std::make_move_iterator(parquet_metadatas.begin()), std::make_move_iterator(parquet_metadatas.end()), @@ -982,6 +987,10 @@ aggregate_reader_metadata::aggregate_reader_metadata( num_rows(calc_num_rows()), num_row_groups(calc_num_row_groups()) { + CUDF_EXPECTS(not per_file_metadata.empty(), + "Encountered an empty vector of parquet sources", + std::invalid_argument); + initialize_internals(use_arrow_schema, has_cols_from_mismatched_srcs); } @@ -1967,6 +1976,8 @@ aggregate_reader_metadata::select_columns( auto const case_sensitive_names = selection_options.case_sensitive_names; auto const selection_mode = selection_options.selection_mode; + auto constexpr root_idx = 0; + // Setup schema lookup helper auto schema_lookup = schema_child_lookup{[&](int const schema_idx, int const src_idx) -> SchemaElement const& { @@ -2011,7 +2022,9 @@ aggregate_reader_metadata::select_columns( auto const dtype = to_data_type(col_type, schema_elem); cudf::io::detail::inline_column_buffer output_col( - dtype, schema_elem.repetition_type == FieldRepetitionType::OPTIONAL); + dtype, + schema_elem.repetition_type == FieldRepetitionType::OPTIONAL or + is_nullable_across_sources(schema_idx)); if (has_list_parent) { output_col.user_data |= PARQUET_COLUMN_BUFFER_FLAG_HAS_LIST_PARENT; } // store the index of this element if inserted in out_col_array nesting.push_back(static_cast(out_col_array.size())); @@ -2052,7 +2065,9 @@ aggregate_reader_metadata::select_columns( auto const element_dtype = to_data_type(element_type, schema_elem); cudf::io::detail::inline_column_buffer element_col( - element_dtype, schema_elem.repetition_type == FieldRepetitionType::OPTIONAL); + element_dtype, + schema_elem.repetition_type == FieldRepetitionType::OPTIONAL or + is_nullable_across_sources(schema_idx)); if (has_list_parent || col_type == type_id::LIST) { element_col.user_data |= PARQUET_COLUMN_BUFFER_FLAG_HAS_LIST_PARENT; } @@ -2087,18 +2102,19 @@ aggregate_reader_metadata::select_columns( }; // Compares two schema elements to be equal except their number of children - auto const equal_to_except_num_children = [selection_mode](SchemaElement const& lhs, - SchemaElement const& rhs) { - // Match by field ID if enabled, otherwise match by name + auto const equal_to_except_num_children = [selection_mode, case_sensitive_names]( + SchemaElement const& lhs, SchemaElement const& rhs) { + // Match by field ID only when it is the selection method, otherwise match by name. Field IDs + // are optional in Parquet and may not be present in all sources. auto const match_schema_by_field_id = selection_mode == column_selection_mode::BY_FIELD_ID; - auto const names_match = + auto const identities_match = (match_schema_by_field_id and lhs.field_id.has_value() and rhs.field_id.has_value()) ? lhs.field_id == rhs.field_id - : lhs.name == rhs.name; + : are_column_paths_equal(lhs.name, rhs.name, case_sensitive_names); return lhs.type == rhs.type and lhs.converted_type == rhs.converted_type and - lhs.type_length == rhs.type_length and names_match and + lhs.type_length == rhs.type_length and identities_match and lhs.decimal_scale == rhs.decimal_scale and - lhs.decimal_precision == rhs.decimal_precision and lhs.field_id == rhs.field_id; + lhs.decimal_precision == rhs.decimal_precision; }; // Maps a projected column's schema_idx in the zeroth per_file_metadata (source) to the @@ -2127,6 +2143,9 @@ aggregate_reader_metadata::select_columns( // Map the schema index from 0th tree (src) to the one in the current (dst) tree. schema_idx_map[src_schema_idx] = dst_schema_idx; + // Mark the field as nullable if it is nullable in the current tree. + propagate_optional_field(src_schema_idx, dst_schema_elem); + // If src_schema_elem is a stub, it does not exist in the column_name_info and column_buffer // hierarchy. So continue on with mapping. if (src_schema_elem.is_stub()) { @@ -2191,6 +2210,29 @@ aggregate_reader_metadata::select_columns( } }; + // Maps a top-level column's schema_idx across the rest of the data sources if we are reading from + // mismatched Parquet sources. `col_name_info` is null when all of the column's children are + // selected. + auto map_column_across_sources = [&](column_name_info const* col_name_info, + std::string const& col_name, + int const src_schema_idx) { + if (per_file_metadata.size() == 1 or schema_idx_maps.empty()) { return; } + + std::for_each( + cuda::counting_iterator{static_cast(1)}, + cuda::counting_iterator{per_file_metadata.size()}, + [&](auto const src_idx) { + // Ensure that each top level column exists in the destination schema tree. + auto const dst_schema_idx = + schema_lookup.find_target_schema_child(root_idx, root_idx, col_name, src_idx); + CUDF_EXPECTS( + dst_schema_idx != -1, + std::format("Encountered missing top-level column '{}' across Parquet sources", col_name), + std::invalid_argument); + map_column(col_name_info, src_schema_idx, dst_schema_idx, src_idx); + }); + }; + std::vector output_column_schemas; // @@ -2215,6 +2257,7 @@ aggregate_reader_metadata::select_columns( auto const& root = get_schema(0); if (not use_names.has_value()) { for (auto const& schema_idx : root.children_idx) { + map_column_across_sources(nullptr, get_schema(schema_idx).name, schema_idx); build_column(nullptr, schema_idx, output_columns, false); output_column_schemas.push_back(schema_idx); } @@ -2330,31 +2373,14 @@ aggregate_reader_metadata::select_columns( } } } + + // Map the column's schema_idx across the rest of the data sources and propagate nullability. for (auto& col : selected_columns) { - auto constexpr root_idx = 0; - auto const& top_level_col_schema_idx = + auto const top_level_col_schema_idx = schema_lookup.find_schema_child_by_name(root_idx, col.name); - bool const valid_column = build_column(&col, top_level_col_schema_idx, output_columns, false); - if (valid_column) { + map_column_across_sources(&col, col.name, top_level_col_schema_idx); + if (build_column(&col, top_level_col_schema_idx, output_columns, false)) { output_column_schemas.push_back(top_level_col_schema_idx); - - // Map the column's schema_idx across the rest of the data sources if required. - if (per_file_metadata.size() > 1 and not schema_idx_maps.empty()) { - std::for_each( - cuda::counting_iterator{static_cast(1)}, - cuda::counting_iterator{per_file_metadata.size()}, - [&](auto const src_idx) { - // Ensure that each top level column exists in the destination schema tree. - auto const dst_col_schema_idx = - schema_lookup.find_target_schema_child(root_idx, root_idx, col.name, src_idx); - CUDF_EXPECTS( - dst_col_schema_idx != -1, - std::format("Encountered missing top-level column '{}' across Parquet sources", - col.name), - std::invalid_argument); - map_column(&col, top_level_col_schema_idx, dst_col_schema_idx, src_idx); - }); - } } } } diff --git a/cpp/src/io/parquet/reader_impl_helpers.hpp b/cpp/src/io/parquet/reader_impl_helpers.hpp index 50279594164d..36ce77a934f9 100644 --- a/cpp/src/io/parquet/reader_impl_helpers.hpp +++ b/cpp/src/io/parquet/reader_impl_helpers.hpp @@ -7,14 +7,17 @@ #include "parquet_gpu.hpp" +#include #include #include #include #include #include -#include +#include +#include #include +#include #include #include #include @@ -22,11 +25,66 @@ #include #include #include +#include #include #include namespace cudf::io::parquet::detail { +/** + * @brief Construct metadatas from inputs using the host worker pool for multiple inputs + * + * All submitted tasks are waited on before any exception is propagated. + * + * @tparam T Metadata construction input type + * @tparam UnaryOp Callable invocable as `op(T const&)` + * + * @throws std::exception The first exception from submitting or running the tasks + * + * @param inputs Metadata construction inputs, one per source + * @param op Operation constructing a metadata object from one input + * @return Constructed metadata objects, in input order + */ +template +[[nodiscard]] auto parallel_construct_metadatas(cudf::host_span inputs, UnaryOp op) +{ + using result_type = std::invoke_result_t; + + std::vector results; + results.reserve(inputs.size()); + + // Avoid using the thread pool for a single input + if (inputs.size() == 1) { + results.emplace_back(op(inputs.front())); + return results; + } + + std::vector> tasks; + tasks.reserve(inputs.size()); + + auto pending_exception = std::exception_ptr{}; + try { + std::transform(inputs.begin(), inputs.end(), std::back_inserter(tasks), [&op](T const& input) { + return cudf::detail::host_worker_pool().submit_task( + [&op, input_ptr = &input] { return op(*input_ptr); }); + }); + } catch (...) { + pending_exception = std::current_exception(); + } + + for (auto& task : tasks) { + try { + results.emplace_back(task.get()); + } catch (...) { + if (not pending_exception) { pending_exception = std::current_exception(); } + } + } + + if (pending_exception) { std::rethrow_exception(pending_exception); } + + return results; +} + /** * @brief page location and size info */ @@ -145,8 +203,13 @@ struct metadata : public FileMetaData { void setup_page_index(cudf::host_span page_index_bytes, int64_t min_offset); + [[nodiscard]] bool is_page_index_setup() const { return is_page_index_setup_; } + protected: void sanitize_schema(); + + private: + bool is_page_index_setup_ = false; }; /** @@ -211,6 +274,7 @@ class aggregate_reader_metadata { std::vector per_file_metadata; std::vector> keyval_maps; std::vector> schema_idx_maps; + std::unordered_set nullable_across_sources; int64_t num_rows; size_type num_row_groups; @@ -240,6 +304,15 @@ class aggregate_reader_metadata { [[nodiscard]] std::vector> init_schema_idx_maps( bool has_cols_from_mismatched_srcs) const; + /** + * @brief Records a schema index as nullable if the corresponding field is nullable in another + * source + * + * @param schema_idx Schema index in the zeroth source + * @param src Corresponding SchemaElement in another source + */ + void propagate_optional_field(int schema_idx, SchemaElement const& src); + /** * @brief Decodes and constructs the arrow schema from the ARROW_SCHEMA_KEY IPC message * in key value metadata section of Parquet file footer @@ -569,6 +642,8 @@ class aggregate_reader_metadata { /** * @brief Checks if a schema index from 0th source is mapped to the specified file index * + * @note Only columns selected by `select_columns` are mapped. + * * @param schema_idx The index of the SchemaElement in the zeroth file. * @param pfm_idx The index of the file (per_file_metadata) to check mappings for. * @@ -579,6 +654,10 @@ class aggregate_reader_metadata { /** * @brief Maps schema index from 0th source file to the specified file index * + * @note Only columns selected by `select_columns` are mapped. + * + * @throws std::out_of_range if `schema_idx` is not mapped to `pfm_idx` + * * @param schema_idx The index of the SchemaElement in the zeroth file. * @param pfm_idx The index of the file (per_file_metadata) to map the schema_idx to. * @@ -586,6 +665,20 @@ class aggregate_reader_metadata { */ [[nodiscard]] int map_schema_index(int schema_idx, int pfm_idx) const; + /** + * @brief Checks if a field that is REQUIRED in the zeroth source is nullable in another source + * + * @note Only columns selected by `select_columns` are tracked. + * + * @param schema_idx The index of the SchemaElement in the zeroth file. + * + * @return True if the field is nullable in a source other than the zeroth one + */ + [[nodiscard]] bool is_nullable_across_sources(int schema_idx) const + { + return nullable_across_sources.contains(schema_idx); + } + /** * @brief Extracts the schema_idx'th SchemaElement from the pfm_idx'th file * @@ -723,6 +816,9 @@ class aggregate_reader_metadata { /** * @brief Filters and reduces down to a selection of columns * + * @note Not thread-safe. Builds the cross-source schema index mappings and the set of fields + * nullable across sources, which are shared by every reader using this metadata object. + * * @param use_names List of paths of column names to select; `nullopt` if user did not select * columns to read * @param filter_columns_names List of paths of column names that are present only in filter diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 9153e1cbb82d..6cc534285af3 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -345,6 +345,7 @@ ConfigureTest( GPUS 1 PERCENT 100 ) +ConfigureTest(COMPACT_PROTOCOL_READER_TEST io/compact_protocol_reader_test.cpp) ConfigureTest( HYBRID_SCAN_TEST io/experimental/hybrid_scan_common.cpp diff --git a/cpp/tests/io/compact_protocol_reader_test.cpp b/cpp/tests/io/compact_protocol_reader_test.cpp new file mode 100644 index 000000000000..0d7bf004dec2 --- /dev/null +++ b/cpp/tests/io/compact_protocol_reader_test.cpp @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#include + +#include +#include +#include +#include +#include + +TEST(CompactProtocolReaderVarintTest, OverflowAtWidthBoundaryU32) +{ + // Fifth group at shift 28: 0x10 << 28 sets bit 32, overflowing uint32_t. + std::vector const bytes{0x80, 0x80, 0x80, 0x80, 0x10}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_THROW(cp.get_u32(), std::overflow_error); +} +TEST(CompactProtocolReaderVarintTest, OverlongSignedThrows) +{ + // get_i32 forwards through get_zigzag to get_varint and propagates overflow_error. + std::vector const bytes{0x80, 0x80, 0x80, 0x80, 0x10}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_THROW(cp.get_i32(), std::overflow_error); +} + +TEST(CompactProtocolReaderVarintTest, WellFormedListHeader) +{ + // Inline size (0x3C: type 0xC, size 3) and escaped size both return {type, size}. + { + std::vector const bytes{0x3C}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_listh(), (std::pair{0xC, 3u})); + } + { + std::vector const bytes{0xF0, 0xAC, 0x02}; // escaped size 300 + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_listh(), (std::pair{0x0, 300u})); + } +} + +TEST(CompactProtocolReaderVarintTest, UnterminatedRunAtEof) +{ + // getb() yields 0 at EOF, ending the loop. A within-width run returns a defined value. + { + std::vector const bytes(3, 0xFF); // shifts 0/7/14 -> 0x1FFFFF + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_u32(), 0x1F'FFFFu); + EXPECT_EQ(cp.bytecount(), static_cast(bytes.size())); + } + { + std::vector const bytes(5, 0xFF); // shifts 0..28 -> 0x7FFFFFFFF + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_u64(), 0x7'FFFF'FFFFUL); + EXPECT_EQ(cp.bytecount(), static_cast(bytes.size())); + } +} + +TEST(CompactProtocolReaderVarintTest, WellFormedValues) +{ + { + // 0x2C | 0x02<<7 = 300. + std::vector const bytes{0x00, 0x7F, 0xAC, 0x02}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_u32(), 0u); + EXPECT_EQ(cp.get_u32(), 127u); + EXPECT_EQ(cp.get_u32(), 300u); + EXPECT_EQ(cp.bytecount(), static_cast(bytes.size())); + } + { + // Maximum five-byte u32: 0x7F | 0x7F<<7 | 0x7F<<14 | 0x7F<<21 | 0x0F<<28. + std::vector const bytes{0xFF, 0xFF, 0xFF, 0xFF, 0x0F}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_u32(), std::numeric_limits::max()); + EXPECT_EQ(cp.bytecount(), static_cast(bytes.size())); + } + { + // Nine-byte u64 reassembling 0x0123456789ABCDEF. + std::vector const bytes{0xEF, 0x9B, 0xAF, 0xCD, 0xF8, 0xAC, 0xD1, 0x91, 0x01}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_u64(), 0x0123'4567'89AB'CDEFUL); + EXPECT_EQ(cp.bytecount(), static_cast(bytes.size())); + } + { + // Maximum ten-byte u64: nine 0x7F groups then 0x01<<63. + std::vector const bytes{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_u64(), std::numeric_limits::max()); + EXPECT_EQ(cp.bytecount(), static_cast(bytes.size())); + } +} + +TEST(CompactProtocolReaderVarintTest, EmptyBuffer) +{ + std::vector const bytes{}; + cudf::io::parquet::detail::CompactProtocolReader cp(bytes.data(), bytes.size()); + EXPECT_EQ(cp.get_u32(), 0u); + EXPECT_EQ(cp.bytecount(), static_cast(bytes.size())); +} + +TEST(CompactProtocolReaderVarintTest, NullBufferWithNonZeroLengthThrows) +{ + // A null base with a positive length has no backing storage; construction rejects it. + EXPECT_THROW(cudf::io::parquet::detail::CompactProtocolReader(nullptr, 4), std::invalid_argument); +} + +TEST(CompactProtocolReaderVarintTest, NullBufferZeroLengthIsDefinedEmpty) +{ + cudf::io::parquet::detail::CompactProtocolReader cp(nullptr, 0); + EXPECT_EQ(cp.get_u32(), 0u); + EXPECT_EQ(cp.bytecount(), static_cast(0)); +} + +CUDF_TEST_PROGRAM_MAIN() diff --git a/cpp/tests/io/experimental/hybrid_scan_multifile_filters_test.cpp b/cpp/tests/io/experimental/hybrid_scan_multifile_filters_test.cpp index 6e7cdad91bde..6895318c4d54 100644 --- a/cpp/tests/io/experimental/hybrid_scan_multifile_filters_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_multifile_filters_test.cpp @@ -7,6 +7,7 @@ #include "tests/io/parquet_common.hpp" #include +#include #include #include @@ -180,17 +181,103 @@ TEST_F(HybridScanMultifileFiltersTest, Metadata) EXPECT_EQ(reader->total_rows_in_row_groups(input_row_group_indices), 2 * rows_per_row_group * num_sources); - // Construct a new reader from a span of existing FileMetaData - auto const reader_with_existing_metadata = - std::make_unique( - cudf::host_span{parquet_metadata}, options); + auto const num_row_groups = parquet_metadata.front().row_groups.size(); - // Check if the new metadata is the same as the existing one - auto const new_metadata = reader_with_existing_metadata->parquet_metadatas(); - ASSERT_EQ(new_metadata.size(), num_sources); - EXPECT_TRUE(std::all_of(new_metadata.begin(), new_metadata.end(), [&](auto const& meta) { - return meta.row_groups.size() == parquet_metadata.front().row_groups.size(); - })); + auto const metadata_matches = [&](auto const& metadatas) { + return metadatas.size() == static_cast(num_sources) and + std::all_of(metadatas.begin(), metadatas.end(), [&](auto const& meta) { + return meta.row_groups.size() == num_row_groups; + }); + }; + + // Copy the existing FileMetaData into a new reader, leaving the source metadata intact + { + auto const reader_from_metadata_span = + std::make_unique( + cudf::host_span{parquet_metadata}, options); + EXPECT_TRUE(metadata_matches(reader_from_metadata_span->parquet_metadatas())); + EXPECT_TRUE(metadata_matches(parquet_metadata)); + } + + // Move the existing FileMetaData into a new reader without copying it. + { + auto const reader_with_existing_metadata = + std::make_unique( + std::move(parquet_metadata), options); + + // Check if the new metadata is the same as the existing one + EXPECT_TRUE(metadata_matches(reader_with_existing_metadata->parquet_metadatas())); + + // Page indexes were moved with the metadata, so the reader must not request them again. + auto const page_index_byte_ranges = reader_with_existing_metadata->page_index_byte_ranges(); + EXPECT_TRUE(std::all_of(page_index_byte_ranges.begin(), + page_index_byte_ranges.end(), + [](auto const& range) { return range.is_empty(); })); + } +} + +TEST_F(HybridScanMultifileFiltersTest, MismatchedNullabilityDoesNotLeakIntoMetadata) +{ + // First source has REQUIRED columns and the second one has reordered and + // nullable ones. + auto const buffer_a = std::get<1>(create_parquet_with_stats()); + auto const buffer_b = std::get<1>(create_parquet_with_stats( + 100, cudf::io::compression_type::AUTO, {"col2", "col0", "col1"}, {2, 0, 1})); + + auto const parquet_buffers = std::vector>{buffer_a, buffer_b}; + auto const source_info = build_source_info(parquet_buffers); + auto inputs = multifile_inputs(source_info); + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + + auto const expected = + cudf::io::read_parquet(cudf::io::parquet_reader_options::builder(source_info) + .allow_mismatched_pq_schemas(true) + .column_names({"col0", "col1", "col2"}) + .build(), + stream, + mr); + + auto const options = + cudf::io::parquet_reader_options::builder().allow_mismatched_pq_schemas(true).build(); + auto const reader = + cudf::io::parquet::experimental::hybrid_scan_multifile{inputs.footer_byte_spans, options}; + + // `col0` is REQUIRED in the first source, and stays REQUIRED in the reported metadata even + // though the second source declares it nullable + auto const col0_repetition_type = [](auto const& metadatas) { + return metadatas.front().schema[1].repetition_type; + }; + + ASSERT_EQ(reader.parquet_metadatas().front().schema[1].name, "col0"); + EXPECT_EQ(col0_repetition_type(reader.parquet_metadatas()), + cudf::io::parquet::FieldRepetitionType::REQUIRED); + + auto const row_groups = reader.all_row_groups(options); + auto column_data = fetch_multisource_device_data( + inputs, reader.all_column_chunks_byte_ranges(row_groups, options), stream, mr); + auto const result = + reader.materialize_all_columns(row_groups, column_data.flat_spans, options, stream, mr); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->view(), result.tbl->view()); + EXPECT_TRUE(result.tbl->view().column(0).nullable()); + EXPECT_TRUE(result.metadata.schema_info[0].is_nullable); + + // Materializing must not have rewritten the reported repetition type + auto metadatas = reader.parquet_metadatas(); + EXPECT_EQ(col0_repetition_type(metadatas), cudf::io::parquet::FieldRepetitionType::REQUIRED); + + // A reader built from that metadata must produce the same (nullable)table. + auto const reader_from_metadata = + cudf::io::parquet::experimental::hybrid_scan_multifile{std::move(metadatas), options}; + auto downstream_column_data = fetch_multisource_device_data( + inputs, reader_from_metadata.all_column_chunks_byte_ranges(row_groups, options), stream, mr); + auto const downstream_result = reader_from_metadata.materialize_all_columns( + row_groups, downstream_column_data.flat_spans, options, stream, mr); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->view(), downstream_result.tbl->view()); + EXPECT_EQ(col0_repetition_type(reader_from_metadata.parquet_metadatas()), + cudf::io::parquet::FieldRepetitionType::REQUIRED); } TEST_F(HybridScanMultifileFiltersTest, EmptySource) diff --git a/cpp/tests/io/experimental/hybrid_scan_multifile_test.cpp b/cpp/tests/io/experimental/hybrid_scan_multifile_test.cpp index 5d3eb9e8083e..80b74e5ba829 100644 --- a/cpp/tests/io/experimental/hybrid_scan_multifile_test.cpp +++ b/cpp/tests/io/experimental/hybrid_scan_multifile_test.cpp @@ -501,3 +501,166 @@ TEST_F(HybridScanMultifileTest, AllColumnsPreservesRequiredNullability) EXPECT_FALSE(result.tbl->view().column(0).nullable()); CUDF_TEST_EXPECT_TABLES_EQUAL(input_table->view(), result.tbl->view()); } + +TEST_F(HybridScanMultifileTest, MismatchedSchemaNullabilityDoesNotLeakIntoMetadata) +{ + // The first source has REQUIRED columns and the second one has the same columns, reordered and + // nullable. The output must be nullable, but the metadata handed back by the reader must keep + // reporting the repetition types that were read from the file footers, both before and after + // materialization. + auto const buffer_a = std::get<1>(create_parquet_with_stats()); + auto const buffer_b = std::get<1>(create_parquet_with_stats( + 100, cudf::io::compression_type::AUTO, {"col2", "col0", "col1"}, {2, 0, 1})); + + auto const parquet_buffers = std::vector>{buffer_a, buffer_b}; + auto const source_info = build_source_info(parquet_buffers); + auto inputs = multifile_inputs(source_info); + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + + auto const expected = + cudf::io::read_parquet(cudf::io::parquet_reader_options::builder(source_info) + .allow_mismatched_pq_schemas(true) + .column_names({"col0", "col1", "col2"}) + .build(), + stream, + mr); + + auto const options = + cudf::io::parquet_reader_options::builder().allow_mismatched_pq_schemas(true).build(); + auto const reader = + cudf::io::parquet::experimental::hybrid_scan_multifile{inputs.footer_byte_spans, options}; + + // `col0` is REQUIRED in the first source, and stays REQUIRED in the reported metadata even + // though the second source declares it nullable + auto const col0_repetition_type = [](auto const& metadatas) { + return metadatas.front().schema[1].repetition_type; + }; + ASSERT_EQ(reader.parquet_metadatas().front().schema[1].name, "col0"); + EXPECT_EQ(col0_repetition_type(reader.parquet_metadatas()), + cudf::io::parquet::FieldRepetitionType::REQUIRED); + + auto const row_groups = reader.all_row_groups(options); + auto column_data = fetch_multisource_device_data( + inputs, reader.all_column_chunks_byte_ranges(row_groups, options), stream, mr); + auto const result = + reader.materialize_all_columns(row_groups, column_data.flat_spans, options, stream, mr); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->view(), result.tbl->view()); + EXPECT_TRUE(result.tbl->view().column(0).nullable()); + EXPECT_TRUE(result.metadata.schema_info[0].is_nullable); + + // Materializing must not have rewritten the reported repetition type + auto metadatas = reader.parquet_metadatas(); + EXPECT_EQ(col0_repetition_type(metadatas), cudf::io::parquet::FieldRepetitionType::REQUIRED); + + // A reader built from that metadata must produce the same table, i.e. no fabricated nullability + // carried over from the first reader + auto const reader_from_metadata = + cudf::io::parquet::experimental::hybrid_scan_multifile{std::move(metadatas), options}; + auto downstream_column_data = fetch_multisource_device_data( + inputs, reader_from_metadata.all_column_chunks_byte_ranges(row_groups, options), stream, mr); + auto const downstream_result = reader_from_metadata.materialize_all_columns( + row_groups, downstream_column_data.flat_spans, options, stream, mr); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->view(), downstream_result.tbl->view()); + EXPECT_EQ(col0_repetition_type(reader_from_metadata.parquet_metadatas()), + cudf::io::parquet::FieldRepetitionType::REQUIRED); +} + +TEST_F(HybridScanMultifileTest, ReadColumnsFromMismatchedSchemas) +{ + // Create two sources with mismatched schemas + auto const buffer_a = std::get<1>(create_parquet_with_stats()); + auto const buffer_b = std::get<1>(create_parquet_with_stats( + 100, cudf::io::compression_type::AUTO, {"col2", "col0", "col1"}, {2, 0, 1})); + + auto const parquet_buffers = std::vector>{buffer_a, buffer_b}; + auto const source_info = build_source_info(parquet_buffers); + auto inputs = multifile_inputs(source_info); + auto const stream = cudf::get_default_stream(); + auto const mr = cudf::get_current_device_resource_ref(); + + // Reading mismatched schemas must be opted into, even without a column projection + EXPECT_THROW(cudf::io::parquet::experimental::hybrid_scan_multifile( + inputs.footer_byte_spans, cudf::io::parquet_reader_options::builder().build()), + cudf::logic_error); + + // Expected table from the regular reader + auto const expected = + cudf::io::read_parquet(cudf::io::parquet_reader_options::builder(source_info) + .allow_mismatched_pq_schemas(true) + .column_names({"col0", "col1", "col2"}) + .build(), + stream, + mr); + + auto options = + cudf::io::parquet_reader_options::builder().allow_mismatched_pq_schemas(true).build(); + auto const reader = + cudf::io::parquet::experimental::hybrid_scan_multifile{inputs.footer_byte_spans, options}; + auto const row_groups = reader.all_row_groups(options); + + // Single step materialize with hybrid scan + { + auto column_data = fetch_multisource_device_data( + inputs, reader.all_column_chunks_byte_ranges(row_groups, options), stream, mr); + auto const result = + reader.materialize_all_columns(row_groups, column_data.flat_spans, options, stream, mr); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->view(), result.tbl->view()); + } + + // Two step materialize with hybrid scan + { + auto literal_value = cudf::numeric_scalar(std::numeric_limits::min()); + auto literal = cudf::ast::literal(literal_value); + auto col_ref = cudf::ast::column_name_reference("col0"); + auto filter = cudf::ast::operation(cudf::ast::ast_operator::GREATER_EQUAL, col_ref, literal); + + options.set_filter(filter); + reader.reset_column_selection(); + + auto row_mask = reader.build_all_true_row_mask(row_groups, stream, mr); + auto row_mask_view = row_mask->mutable_view(); + + auto filter_column_chunks = fetch_multisource_device_data( + inputs, reader.filter_column_chunks_byte_ranges(row_groups, options), stream, mr); + auto const filter_result = reader.materialize_filter_columns(row_groups, + filter_column_chunks.flat_spans, + row_mask_view, + use_data_page_mask::NO, + options, + stream, + mr); + + auto payload_column_chunks = fetch_multisource_device_data( + inputs, reader.payload_column_chunks_byte_ranges(row_groups, options), stream, mr); + auto const payload_result = reader.materialize_payload_columns(row_groups, + payload_column_chunks.flat_spans, + row_mask_view, + use_data_page_mask::NO, + options, + stream, + mr); + + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->select({0}), filter_result.tbl->view()); + CUDF_TEST_EXPECT_TABLES_EQUAL(expected.tbl->select({1, 2}), payload_result.tbl->view()); + } +} + +TEST_F(HybridScanMultifileTest, EmptySources) +{ + // Arrow schema is applied during metadata construction, so make sure empty inputs are rejected + // before any metadata is touched + auto const options = cudf::io::parquet_reader_options::builder().use_arrow_schema(true).build(); + + EXPECT_THROW(cudf::io::parquet::experimental::hybrid_scan_multifile( + cudf::host_span const>{}, options), + std::invalid_argument); + EXPECT_THROW(cudf::io::parquet::experimental::hybrid_scan_multifile( + cudf::host_span{}, options), + std::invalid_argument); + EXPECT_THROW(cudf::io::parquet::experimental::hybrid_scan_multifile( + std::vector{}, options), + std::invalid_argument); +} diff --git a/cpp/tests/io/parquet_reader_test.cpp b/cpp/tests/io/parquet_reader_test.cpp index 8f61d2f2c07e..824b867da7bb 100644 --- a/cpp/tests/io/parquet_reader_test.cpp +++ b/cpp/tests/io/parquet_reader_test.cpp @@ -724,23 +724,24 @@ TEST_F(ParquetReaderTest, SelectMismatchedStructChildByFieldId) .metadata(std::move(metadata_a)); cudf::io::write_parquet(write_args_a); - auto y_b = cudf::test::fixed_width_column_wrapper{40, 50}; + auto y_b = cudf::test::fixed_width_column_wrapper{{40, 50}, {true, false}}; auto x_b = cudf::test::fixed_width_column_wrapper{4, 5}; auto struct_b = cudf::test::structs_column_wrapper{{y_b, x_b}, {true, true}}.release(); cudf::table_view const table_b{{*struct_b}}; auto path_b = temp_env->get_temp_filepath("SelectNestedFieldIdChildOrderB.parquet"); cudf::io::table_input_metadata metadata_b(table_b); - metadata_b.column_metadata[0].set_name("record").set_parquet_field_id(1); - metadata_b.column_metadata[0].child(0).set_name("y").set_parquet_field_id(3); - metadata_b.column_metadata[0].child(1).set_name("x").set_parquet_field_id(2); + metadata_b.column_metadata[0].set_name("renamed_record").set_parquet_field_id(1); + metadata_b.column_metadata[0].child(0).set_name("renamed_y").set_parquet_field_id(3); + metadata_b.column_metadata[0].child(1).set_name("renamed_x").set_parquet_field_id(2); auto write_args_b = cudf::io::parquet_writer_options::builder(cudf::io::sink_info{path_b}, table_b) .metadata(std::move(metadata_b)); cudf::io::write_parquet(write_args_b); auto expected_x = cudf::test::fixed_width_column_wrapper{1, 2, 3, 4, 5}; - auto expected_y = cudf::test::fixed_width_column_wrapper{10, 20, 30, 40, 50}; + auto expected_y = cudf::test::fixed_width_column_wrapper{ + {10, 20, 30, 40, 50}, {true, true, true, true, false}}; auto expected_struct = cudf::test::structs_column_wrapper{{expected_x, expected_y}, {true, true, true, true, true}} .release(); @@ -5650,6 +5651,17 @@ TEST_F(ParquetReaderTest, LateBindSourceInfo) CUDF_TEST_EXPECT_TABLES_EQUAL(result.tbl->view(), expected->view()); } +TEST_F(ParquetReaderTest, EmptySourcesWithArrowSchema) +{ + auto sources = std::vector>{}; + auto file_metadatas = std::vector{}; + auto const options = cudf::io::parquet_reader_options::builder(cudf::io::source_info{}) + .use_arrow_schema(true) + .build(); + EXPECT_THROW(cudf::io::read_parquet(std::move(sources), std::move(file_metadatas), options), + std::invalid_argument); +} + TEST_F(ParquetReaderTest, InvalidFooterMagic) { auto const expected = create_random_fixed_table(4, 4, false); @@ -6530,6 +6542,30 @@ TEST_F(ParquetReaderTest, NestedMismatchedSchemaColumnValidation) EXPECT_THROW(cudf::io::read_parquet(opts), std::invalid_argument); } } + +TEST_F(ParquetReaderTest, CaseInsensitiveMismatchedSchemasPropagateNullability) +{ + auto const required = cudf::test::fixed_width_column_wrapper{1, 2, 3}; + auto const optional = cudf::test::fixed_width_column_wrapper{{4, 5}, {true, false}}; + auto const required_path = + write_parquet_temp_file(cudf::table_view{{required}}, "CaseRequired.parquet", {"column"}); + auto const optional_path = + write_parquet_temp_file(cudf::table_view{{optional}}, "CaseOptional.parquet", {"COLUMN"}); + + auto const options = + cudf::io::parquet_reader_options::builder(cudf::io::source_info{{required_path, optional_path}}) + .allow_mismatched_pq_schemas(true) + .case_sensitive_names(false) + .column_names({"column"}) + .build(); + + // A non-nullable column in the first source but nullable in another must be read as nullable. + auto const expected = cudf::test::fixed_width_column_wrapper{ + {1, 2, 3, 4, 5}, {true, true, true, true, false}}; + auto const result = cudf::io::read_parquet(options); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(result.tbl->view().column(0), expected); +} + namespace { /** diff --git a/dependencies.yaml b/dependencies.yaml index b6b263bdba03..675aac3e783d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -714,7 +714,7 @@ files: test_cudf_polars_compat: output: none matrix: - polars_compat_version: ["1.35", "1.36", "1.37", "1.38", "1.39", "1.40", "1.41", "1.42"] + polars_compat_version: ["1.35", "1.36", "1.37", "1.38", "1.39", "1.40", "1.41", "1.42", "1.43", "1.44"] includes: - test_cudf_polars_compat channels: @@ -1098,7 +1098,7 @@ dependencies: # 'nvidia-ml-py' provides the 'pynvml' module - &nvidia_ml_py nvidia-ml-py>=12 - packaging - - polars>=1.35,<1.43 + - polars>=1.35,<1.45 - typing_extensions>=4.0.0 run_cudf_polars_dask: common: @@ -1939,5 +1939,13 @@ dependencies: polars_compat_version: "1.42" packages: - polars==1.42.* + - matrix: + polars_compat_version: "1.43" + packages: + - polars==1.43.* + - matrix: + polars_compat_version: "1.44" + packages: + - polars==1.44.* - matrix: packages: diff --git a/python/cudf/CMakeLists.txt b/python/cudf/CMakeLists.txt index 5225e173a05d..4b1d183530ba 100644 --- a/python/cudf/CMakeLists.txt +++ b/python/cudf/CMakeLists.txt @@ -29,10 +29,5 @@ find_package(cudf "${RAPIDS_VERSION}" REQUIRED) include(rapids-cython-core) rapids_cython_init() -# Enable Cython's translation cache when CI supplies a persistent cache directory. -if(DEFINED ENV{CYTHON_CACHE_DIR} AND NOT "$ENV{CYTHON_CACHE_DIR}" STREQUAL "") - list(PREPEND CYTHON_FLAGS "--cache") -endif() - add_subdirectory(cudf/_lib) add_subdirectory(udf_cpp) diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py b/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py index 7f1b9cc33687..c08579418dae 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/boolean.py @@ -560,14 +560,13 @@ def do_evaluate( ), dtype=self.dtype, ) - return Column( - plc.Column.from_scalar( - plc.Scalar.from_py(py_val=False, stream=df.stream), - needles.size, - stream=df.stream, - ), - dtype=self.dtype, + base = plc.Column.from_scalar( + plc.Scalar.from_py(py_val=False, stream=df.stream), + needles.size, + stream=df.stream, ) + out = base.with_mask(needles.obj.null_mask(), needles.null_count) + return Column(out, dtype=self.dtype) elif self.name is BooleanFunction.Name.IsSorted: (column,) = columns (descending, nulls_last) = self.options diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/string.py b/python/cudf_polars/cudf_polars/dsl/expressions/string.py index 23be90aed6a8..a04b68b4e922 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/string.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/string.py @@ -71,7 +71,10 @@ class Name(IntEnum): Extract = auto() ExtractAll = auto() ExtractGroups = auto() + ExtractMany = auto() Find = auto() + FindMany = auto() + Format = auto() Head = auto() HexDecode = auto() HexEncode = auto() diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/struct.py b/python/cudf_polars/cudf_polars/dsl/expressions/struct.py index 18885ec9724a..4bcadcc3c0e3 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/struct.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/struct.py @@ -33,6 +33,7 @@ class Name(IntEnum): """Internal and picklable representation of polars' `StructFunction`.""" FieldByName = auto() + DropFields = auto() RenameFields = auto() PrefixFields = auto() SuffixFields = auto() diff --git a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py index f899cb0a8e93..529456191d84 100644 --- a/python/cudf_polars/cudf_polars/dsl/expressions/unary.py +++ b/python/cudf_polars/cudf_polars/dsl/expressions/unary.py @@ -224,6 +224,13 @@ def __init__( raise NotImplementedError(f"Unary function {name=}") # pragma: no cover if self.name == "index_of" and plc.traits.is_nested(children[0].dtype.plc_type): raise NotImplementedError("index_of on nested types is not supported") + if ( + self.name in UnaryFunction._OP_MAPPING + or self.name in UnaryFunction._supported_math_fns + or self.name == "pct_change" + ) and plc.traits.is_nested(children[0].dtype.plc_type): + # TODO: polars should fail ahead of us + raise NotImplementedError(f"{name} on nested types is not supported") if self.name == "entropy" and not plc.traits.is_numeric_not_bool( children[0].dtype.plc_type ): diff --git a/python/cudf_polars/cudf_polars/dsl/ir.py b/python/cudf_polars/cudf_polars/dsl/ir.py index 4941391032ac..bc6534713ce8 100644 --- a/python/cudf_polars/cudf_polars/dsl/ir.py +++ b/python/cudf_polars/cudf_polars/dsl/ir.py @@ -817,11 +817,18 @@ def __init__( # column names, we would need to do file introspection to infer the number # of columns so column projection works right. reader_schema = self.reader_options.get("schema") - if not ( - reader_schema - and isinstance(schema, dict) - and "fields" in reader_schema - ): + has_new_columns = bool( + ( + reader_schema + and isinstance(schema, dict) + and "fields" in reader_schema + ) + # polars 1.43 added a separate "column_names_overwrite" + # for headerless new_columns; "schema" still covers other + # renaming mechanisms (e.g. with_column_names) unchanged. + or self.reader_options.get("column_names_overwrite") + ) + if not has_new_columns: raise NotImplementedError( "Reading CSV without header requires user-provided column names via new_columns" ) @@ -998,6 +1005,18 @@ def read_csv_header( if reader_options["schema"] is not None: # Reader schema provides names column_names = list(reader_options["schema"]["fields"].keys()) + elif reader_options.get("column_names_overwrite"): + # polars 1.43 added "column_names_overwrite" for headerless + # new_columns; "schema" (above) still covers other renaming + # mechanisms (e.g. with_column_names) unchanged. + # "column_names_overwrite" only lists the overridden names, + # not any trailing columns padded with polars' default + # "column_N" names. Read the full, resolved set of names from + # schema instead. Exclude the row index, which isn't a + # column the CSV reader produces. + column_names = [ + name for name in schema if row_index is None or name != row_index[0] + ] else: # file provides column names column_names = None diff --git a/python/cudf_polars/cudf_polars/dsl/to_ast.py b/python/cudf_polars/cudf_polars/dsl/to_ast.py index edd33c87efce..0eafd618faeb 100644 --- a/python/cudf_polars/cudf_polars/dsl/to_ast.py +++ b/python/cudf_polars/cudf_polars/dsl/to_ast.py @@ -182,7 +182,7 @@ def _(node: expr.BinOp, self: Transformer) -> plc_expr.Expression: if c1.dtype != c2.dtype: if isinstance(c1, expr.Literal): # pragma: no cover c1 = c1.astype(c2.dtype) - elif isinstance(c2, expr.Literal): + elif isinstance(c2, expr.Literal): # pragma: no cover c2 = c2.astype(c1.dtype) elif ( isinstance(c1, (expr.Col, expr.ColRef)) and c1.dtype.id() in _DECIMAL_IDS diff --git a/python/cudf_polars/cudf_polars/dsl/translate.py b/python/cudf_polars/cudf_polars/dsl/translate.py index 23642bf64196..d11102f0283a 100644 --- a/python/cudf_polars/cudf_polars/dsl/translate.py +++ b/python/cudf_polars/cudf_polars/dsl/translate.py @@ -41,6 +41,8 @@ POLARS_VERSION_LT_140, POLARS_VERSION_LT_141, POLARS_VERSION_LT_142, + POLARS_VERSION_LT_143, + POLARS_VERSION_LT_144, ) if TYPE_CHECKING: @@ -69,7 +71,10 @@ def _align_decimal_float_for_comparison( """ has_decimal = any(plc.traits.is_fixed_point(op.dtype.plc_type) for op in operands) has_float = any(plc.traits.is_floating_point(op.dtype.plc_type) for op in operands) - if has_decimal and has_float: + if has_decimal and has_float: # pragma: no cover + # Polars now inserts this cast itself under the latest supported + # version (coverage only runs against latest), but older supported + # versions still need this workaround. f64 = DataType(pl.Float64()) return tuple( expr.Cast(f64, False, op) # noqa: FBT003 @@ -153,6 +158,28 @@ def _unsupported_fill_over_window(value: expr.Expr) -> bool: ) +def _is_len_sum_uint128_node(visitor: NodeTraverser, node: Any) -> bool: + """ + Whether ``node`` is part of polars' ``col("len").cast(UInt128).sum()``. + + polars rewrites ``concat(...).select(len())`` into + ``col("len").cast(UInt128).sum().cast(IDX_DTYPE)``. Both the ``sum`` and + its ``"len"`` cast independently report a ``UInt128`` dtype during + translation, so both must be recognized here. + """ + # TODO: this matches the exact shape of that one rewrite, not UInt128 in + # general; if polars changes it, or introduces UInt128 elsewhere, this + # will stop matching (and fall back to CPU / raise cleanly, not silently + # misbehave, since general UInt128 use is still unconditionally rejected + # elsewhere). See https://github.com/NVIDIA/cudf/issues/24108. + if isinstance(node, plrs._expr_nodes.Agg): + return node.name == "sum" + if isinstance(node, plrs._expr_nodes.Cast): + child = visitor.view_expression(node.expr) + return isinstance(child, plrs._expr_nodes.Column) and child.name == "len" + return False # pragma: no cover + + class Translator: """ Translates polars-internal IR nodes and expressions to our representation. @@ -207,7 +234,7 @@ def translate_ir(self, *, n: int | None = None) -> ir.IR: # IR is versioned with major.minor, minor is bumped for backwards # compatible changes (e.g. adding new nodes), major is bumped for # incompatible changes (e.g. renaming nodes). - if (version := self.visitor.version()) >= (14, 4): + if (version := self.visitor.version()) >= (14, 8): e = NotImplementedError( f"No support for polars IR {version=}" ) # pragma: no cover; no such version for now. @@ -294,7 +321,20 @@ def translate_expr( to determine if the query is supported. """ node = self.visitor.view_expression(n) - dtype = DataType(self.visitor.get_dtype(n)) + polars_dtype = self.visitor.get_dtype(n) + if isinstance(polars_dtype, pl.UInt128) and _is_len_sum_uint128_node( + self.visitor, node + ): + # libcudf has no 128-bit integer type (size_type is 32-bit today; + # see https://github.com/NVIDIA/cudf/issues/13159). polars + # rewrites concat(...).select(len()) into + # col("len").cast(UInt128).sum().cast(IDX_DTYPE), widening before + # the sum so it can't overflow, then narrowing back down. This + # value is never materialized as real UInt128 data, so represent + # it as UInt64 instead: no libcudf table can hold anywhere near + # 2**64 rows, so the sum can't overflow it either. + polars_dtype = pl.UInt64() + dtype = DataType(polars_dtype) is_array_passthrough = ( allow_array_passthrough and isinstance(dtype.polars_type, pl.Array) @@ -812,6 +852,11 @@ def _( node: plrs._ir_nodes.MergeSorted, translator: Translator, schema: Schema ) -> ir.IR: key = node.key + if not POLARS_VERSION_LT_143: + # node.key became a list of keys in polars 1.43. + if len(key) != 1: + raise NotImplementedError("Merging on multiple keys is not supported") + key = key[0] inp_left = translator.translate_ir(n=node.input_left) inp_right = translator.translate_ir(n=node.input_right) return ir.MergeSorted( @@ -1072,6 +1117,12 @@ def _( return expr.Cast(dtype, True, result_expr) # noqa: FBT003 return result_expr elif isinstance(name, plrs._expr_nodes.StructFunction): + if ( + not POLARS_VERSION_LT_144 + and name == plrs._expr_nodes.StructFunction.RenameFields + ): + (new_field_names,) = options + options = (tuple(new_field_names),) return expr.StructFunction( dtype, expr.StructFunction.Name.from_polars(name), diff --git a/python/cudf_polars/cudf_polars/engine/core.py b/python/cudf_polars/cudf_polars/engine/core.py index f9da3505c1c6..b3f00976b630 100644 --- a/python/cudf_polars/cudf_polars/engine/core.py +++ b/python/cudf_polars/cudf_polars/engine/core.py @@ -24,6 +24,9 @@ from rapidsmpf.coll import AllGather from rapidsmpf.config import Options, get_environment_variables from rapidsmpf.memory.packed_data import PackedData +from rapidsmpf.memory.pinned_memory_resource import ( + is_pinned_memory_resources_supported, +) from rapidsmpf.statistics import Statistics from rapidsmpf.streaming.core.actor import run_actor_network @@ -211,8 +214,9 @@ def resolve_rapidsmpf_options(rapidsmpf_options: Options | None) -> Options: - ``num_streaming_threads=4``: moderate worker count for the rapidsmpf streaming runtime, shared across frontends. - - ``pinned_memory=true``, ``pinned_initial_pool_size=0``: pinned host - memory enabled by default. + - ``pinned_memory``, ``pinned_initial_pool_size=0``: pinned host memory + enabled by default, but only on systems that support it (CUDA 12.6+ + with async memory pool support). Parameters ---------- @@ -228,10 +232,13 @@ def resolve_rapidsmpf_options(rapidsmpf_options: Options | None) -> Options: if rapidsmpf_options is None: rapidsmpf_options = Options(get_environment_variables()) + pinned_memory_default = ( + "true" if is_pinned_memory_resources_supported() else "false" + ) rapidsmpf_options.insert_if_absent( { "num_streaming_threads": "4", - "pinned_memory": "true", + "pinned_memory": pinned_memory_default, "pinned_initial_pool_size": "0", } ) @@ -294,12 +301,24 @@ class StreamingEngine(pl.GPUEngine): destruction and context manager exit must occur on the thread that created the instance. - Creating an engine sets the kvikio remote I/O backend to ``EASY_THREADPOOL`` - and configures its thread pool (default 256 threads). Because kvikio's pool - is a global singleton, this blocks any concurrent kvikio IO in the process - until in-flight IO completes and overrides any prior ``kvikio.defaults.set(...)`` - calls. Use the ``kvikio_nthreads`` executor option or the ``KVIKIO_NTHREADS`` - environment variable to control the thread count. + Creating an engine sets the kvikio remote I/O backend to + ``kvikio.RemoteIOBackend.MULTI_POLL`` by default (see the + ``kvikio_remote_io_backend`` executor option), along with the + ``kvikio_task_size`` executor option (16 MiB under ``MULTI_POLL``, 64 MiB + under ``EASY_THREADPOOL``). Because kvikio's configuration is a global + singleton, this overrides mutable prior ``kvikio.defaults.set(...)`` calls + made in the process. The ``MULTI_POLL`` reactor settings are process-lifetime + values: after the first remote I/O, subsequent engines must use the same + values. When the backend is ``EASY_THREADPOOL``, engine creation + also configures kvikio's thread pool (default 256 threads), which blocks + any concurrent kvikio IO in the process until in-flight IO completes. Use + the ``kvikio_nthreads`` executor option or the ``KVIKIO_NTHREADS`` + environment variable to control the thread count. Under ``MULTI_POLL``, + cudf-polars does not resolve a thread-pool size at all (kvikio itself may + still honor ``KVIKIO_NTHREADS`` via its own deferred default); remote I/O + concurrency is instead controlled by the ``kvikio_reactor_count``, + ``kvikio_reactor_dispatch``, and ``kvikio_request_ceiling`` executor + options. Parameters ---------- diff --git a/python/cudf_polars/cudf_polars/engine/dask.py b/python/cudf_polars/cudf_polars/engine/dask.py index bcce9bad3550..4dec30c938d4 100644 --- a/python/cudf_polars/cudf_polars/engine/dask.py +++ b/python/cudf_polars/cudf_polars/engine/dask.py @@ -15,6 +15,7 @@ import distributed import distributed.system +import kvikio import pynvml import ucxx._lib.libucxx as ucx_api @@ -58,8 +59,7 @@ DaskContext, MemoryResourceConfig, configure_kvikio, - resolve_kvikio_nthreads, - resolve_kvikio_statistics, + resolve_kvikio_executor_options, ) if TYPE_CHECKING: @@ -338,8 +338,14 @@ def _setup_worker( worker_ids: list[uuid.UUID], engine_id: uuid.UUID, num_py_executors: int, - kvikio_nthreads: int, + kvikio_nthreads: int | None, kvikio_statistics: bool, + kvikio_remote_io_backend: kvikio.RemoteIOBackend, + kvikio_task_size: int, + kvikio_bounce_buffer_bytes: int, + kvikio_reactor_count: int, + kvikio_reactor_dispatch: kvikio.RemoteReactorDispatch, + kvikio_request_ceiling: int, quent_context: cudf_polars.quent.QuentContext | None, dask_worker: distributed.Worker | None = None, ) -> None: @@ -376,9 +382,22 @@ def _setup_worker( num_py_executors Number of Python executors to use for this worker. kvikio_nthreads - Number of kvikio threads to configure on this worker process. + Number of kvikio threads to configure on this worker process. ``None`` + defers to kvikio's own built-in default. kvikio_statistics Whether to collect KvikIO I/O statistics on this worker. + kvikio_remote_io_backend + The kvikio remote I/O backend to configure on this worker process. + kvikio_task_size + Size, in bytes, of the kvikio task size to configure on this worker process. + kvikio_bounce_buffer_bytes + Size, in bytes, of the kvikio bounce buffer to configure on this worker process. + kvikio_reactor_count + Number of ``MULTI_POLL`` reactor threads to configure on this worker process. + kvikio_reactor_dispatch + ``MULTI_POLL`` reactor dispatch policy to configure on this worker process. + kvikio_request_ceiling + ``MULTI_POLL`` concurrent-request ceiling to configure on this worker process. quent_context Quent context to use for this worker, if quent is enabled. @@ -391,7 +410,15 @@ def _setup_worker( if mp_ctx is None: # Non-root worker: create communicator now. bind_to_gpu(hardware_binding) - configure_kvikio(kvikio_nthreads) + configure_kvikio( + kvikio_nthreads, + remote_io_backend=kvikio_remote_io_backend, + task_size=kvikio_task_size, + bounce_buffer_bytes=kvikio_bounce_buffer_bytes, + reactor_count=kvikio_reactor_count, + reactor_dispatch=kvikio_reactor_dispatch, + request_ceiling=kvikio_request_ceiling, + ) memory_resource_config = ( memory_resource_config or MemoryResourceConfig.default() ) @@ -412,7 +439,15 @@ def _setup_worker( base_mr = mp_ctx.base_mr comm = mp_ctx.comm statistics = mp_ctx.statistics - configure_kvikio(kvikio_nthreads) + configure_kvikio( + kvikio_nthreads, + remote_io_backend=kvikio_remote_io_backend, + task_size=kvikio_task_size, + bounce_buffer_bytes=kvikio_bounce_buffer_bytes, + reactor_count=kvikio_reactor_count, + reactor_dispatch=kvikio_reactor_dispatch, + request_ceiling=kvikio_request_ceiling, + ) barrier(comm) worker_id = worker_ids[comm.rank] @@ -508,8 +543,14 @@ def _reset_worker( rapidsmpf_options_as_bytes: bytes, *, uid: str, - kvikio_nthreads: int, + kvikio_nthreads: int | None, kvikio_statistics: bool, + kvikio_remote_io_backend: kvikio.RemoteIOBackend, + kvikio_task_size: int, + kvikio_bounce_buffer_bytes: int, + kvikio_reactor_count: int, + kvikio_reactor_dispatch: kvikio.RemoteReactorDispatch, + kvikio_request_ceiling: int, dask_worker: distributed.Worker | None = None, ) -> None: """ @@ -525,14 +566,35 @@ def _reset_worker( uid Cluster instance identifier used to look up the per-worker context. kvikio_nthreads - Number of kvikio threads to configure on this worker process. + Number of kvikio threads to configure on this worker process. ``None`` + defers to kvikio's own built-in default. kvikio_statistics Whether to collect KvikIO I/O statistics on this worker. + kvikio_remote_io_backend + The kvikio remote I/O backend to configure on this worker process. + kvikio_task_size + Size, in bytes, of the kvikio task size to configure on this worker process. + kvikio_bounce_buffer_bytes + Size, in bytes, of the kvikio bounce buffer to configure on this worker process. + kvikio_reactor_count + Number of ``MULTI_POLL`` reactor threads to configure on this worker process. + kvikio_reactor_dispatch + ``MULTI_POLL`` reactor dispatch policy to configure on this worker process. + kvikio_request_ceiling + ``MULTI_POLL`` concurrent-request ceiling to configure on this worker process. dask_worker Injected by ``distributed`` when called via :meth:`distributed.Client.run`. """ assert dask_worker is not None - configure_kvikio(kvikio_nthreads) + configure_kvikio( + kvikio_nthreads, + remote_io_backend=kvikio_remote_io_backend, + task_size=kvikio_task_size, + bounce_buffer_bytes=kvikio_bounce_buffer_bytes, + reactor_count=kvikio_reactor_count, + reactor_dispatch=kvikio_reactor_dispatch, + request_ceiling=kvikio_request_ceiling, + ) attr = f"_cudf_polars_mp_context_{uid}" mp_ctx: _WorkerContext | None = getattr(dask_worker, attr, None) if mp_ctx is None: @@ -951,13 +1013,7 @@ def __init__( executor_options: dict[str, Any] | None = None, engine_options: dict[str, Any] | None = None, ) -> None: - executor_options = executor_options or {} - executor_options.setdefault( - "kvikio_nthreads", resolve_kvikio_nthreads(executor_options) - ) - executor_options.setdefault( - "kvikio_statistics", resolve_kvikio_statistics(executor_options) - ) + executor_options = resolve_kvikio_executor_options(executor_options or {}) engine_options = engine_options or {} quent_context: cudf_polars.quent.QuentContext | None = executor_options.get( @@ -1074,6 +1130,12 @@ def __init__( num_py_executors=executor_options.get("num_py_executors", 8), kvikio_nthreads=executor_options["kvikio_nthreads"], kvikio_statistics=executor_options["kvikio_statistics"], + kvikio_remote_io_backend=executor_options["kvikio_remote_io_backend"], + kvikio_task_size=executor_options["kvikio_task_size"], + kvikio_bounce_buffer_bytes=executor_options["kvikio_bounce_buffer_bytes"], + kvikio_reactor_count=executor_options["kvikio_reactor_count"], + kvikio_reactor_dispatch=executor_options["kvikio_reactor_dispatch"], + kvikio_request_ceiling=executor_options["kvikio_request_ceiling"], ) dask_ctx = DaskContext( @@ -1111,16 +1173,16 @@ def _reset( ) executor_options = executor_options or {} existing_executor_options = self.config.get("executor_options", {}) - if isinstance(existing_executor_options, dict): - existing_quent_context = existing_executor_options.get("quent_context") - if existing_quent_context is not None: - executor_options.setdefault("quent_context", existing_quent_context) - existing_kvikio_nthreads = existing_executor_options.get("kvikio_nthreads") - if existing_kvikio_nthreads is not None: - executor_options.setdefault("kvikio_nthreads", existing_kvikio_nthreads) - executor_options.setdefault( - "kvikio_statistics", resolve_kvikio_statistics(executor_options) - ) + if not isinstance(existing_executor_options, dict): + existing_executor_options = {} + existing_quent_context = existing_executor_options.get("quent_context") + if existing_quent_context is not None: + executor_options.setdefault("quent_context", existing_quent_context) + if "kvikio_nthreads" in existing_executor_options: + executor_options.setdefault( + "kvikio_nthreads", existing_executor_options["kvikio_nthreads"] + ) + executor_options = resolve_kvikio_executor_options(executor_options) engine_options = engine_options or {} self.rapidsmpf_options = resolve_rapidsmpf_options(rapidsmpf_options) @@ -1137,6 +1199,14 @@ def _reset( uid=ctx.rapidsmpf_id, kvikio_nthreads=executor_options["kvikio_nthreads"], kvikio_statistics=executor_options["kvikio_statistics"], + kvikio_remote_io_backend=executor_options["kvikio_remote_io_backend"], + kvikio_task_size=executor_options["kvikio_task_size"], + kvikio_bounce_buffer_bytes=executor_options[ + "kvikio_bounce_buffer_bytes" + ], + kvikio_reactor_count=executor_options["kvikio_reactor_count"], + kvikio_reactor_dispatch=executor_options["kvikio_reactor_dispatch"], + kvikio_request_ceiling=executor_options["kvikio_request_ceiling"], ), rapidsmpf_options_as_bytes, ) @@ -1331,8 +1401,11 @@ def _run_by_rank( ) return dict(sorted(results.values(), key=lambda pair: pair[0])) + # TODO: adopt polars' Engine.execute(lf, *, optimizations) contract + # (added in polars>=1.43) so we can return our own result type from + # LazyFrame.execute(engine=...) too (See https://github.com/NVIDIA/cudf/issues/22917). @unstable() - def execute(self, lf: pl.LazyFrame) -> PersistedQueryResult: + def execute(self, lf: pl.LazyFrame) -> PersistedQueryResult: # type: ignore[override] """ Execute a :class:`~polars.LazyFrame` and return a distributed result. diff --git a/python/cudf_polars/cudf_polars/engine/options.py b/python/cudf_polars/cudf_polars/engine/options.py index cae2a26202ad..b90a1be4f272 100644 --- a/python/cudf_polars/cudf_polars/engine/options.py +++ b/python/cudf_polars/cudf_polars/engine/options.py @@ -12,6 +12,8 @@ import textwrap from typing import TYPE_CHECKING, Any, Literal +import kvikio + from rapidsmpf.config import Options from rapidsmpf.utils.string import parse_boolean @@ -42,6 +44,14 @@ ] +def _parse_remote_io_backend(value: str) -> kvikio.RemoteIOBackend: + return kvikio.RemoteIOBackend[value.upper()] + + +def _parse_reactor_dispatch(value: str) -> kvikio.RemoteReactorDispatch: + return kvikio.RemoteReactorDispatch[value.upper()] + + def _opt( category: str, env_var: str | None = None, @@ -213,6 +223,42 @@ class StreamingOptions: Env: ``CUDF_POLARS__EXECUTOR__KVIKIO_STATISTICS``. Default: ``False``. Category: executor. + kvikio_remote_io_backend + The kvikio remote I/O backend. + Env: ``CUDF_POLARS__EXECUTOR__KVIKIO_REMOTE_IO_BACKEND``. + Default: ``kvikio.RemoteIOBackend.MULTI_POLL``. + Category: executor. + kvikio_task_size + Size, in bytes, of the chunks kvikio splits reads into for parallel + dispatch. Applies to local and remote I/O under both backends. + Env: ``CUDF_POLARS__EXECUTOR__KVIKIO_TASK_SIZE``. + Default: ``16777216`` (16 MiB) for the ``MULTI_POLL`` backend, + ``67108864`` (64 MiB) for ``EASY_THREADPOOL``. + Category: executor. + kvikio_bounce_buffer_bytes + Size, in bytes, of the kvikio bounce buffer used to stage host memory + for device-memory transfers. Applies to local and remote I/O under + both backends (not specific to ``MULTI_POLL``). + Env: ``CUDF_POLARS__EXECUTOR__KVIKIO_BOUNCE_BUFFER_BYTES``. + Default: ``16777216`` (16 MiB). + Category: executor. + kvikio_reactor_count + Number of reactor threads used by the ``MULTI_POLL`` remote I/O backend. + Env: ``CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_COUNT``. + Default: ``24``. + Category: executor. + kvikio_reactor_dispatch + How sub-ranges of one read are distributed across reactor threads under + the ``MULTI_POLL`` remote I/O backend. + Env: ``CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_DISPATCH``. + Default: ``kvikio.RemoteReactorDispatch.PER_CHUNK``. + Category: executor. + kvikio_request_ceiling + Maximum number of concurrent in-flight requests across all reactor + threads under the ``MULTI_POLL`` remote I/O backend. 0 means unlimited. + Env: ``CUDF_POLARS__EXECUTOR__KVIKIO_REQUEST_CEILING``. + Default: ``256``. + Category: executor. max_concurrent_io_tasks Maximum concurrent IO tasks for each scan node. Env: ``CUDF_POLARS__EXECUTOR__MAX_CONCURRENT_IO_TASKS``. @@ -347,6 +393,28 @@ class StreamingOptions: kvikio_statistics: bool | Unspecified = _opt( "executor", "CUDF_POLARS__EXECUTOR__KVIKIO_STATISTICS", parse_boolean ) + kvikio_remote_io_backend: kvikio.RemoteIOBackend | Unspecified = _opt( + "executor", + "CUDF_POLARS__EXECUTOR__KVIKIO_REMOTE_IO_BACKEND", + _parse_remote_io_backend, + ) + kvikio_task_size: int | Unspecified = _opt( + "executor", "CUDF_POLARS__EXECUTOR__KVIKIO_TASK_SIZE", int + ) + kvikio_bounce_buffer_bytes: int | Unspecified = _opt( + "executor", "CUDF_POLARS__EXECUTOR__KVIKIO_BOUNCE_BUFFER_BYTES", int + ) + kvikio_reactor_count: int | Unspecified = _opt( + "executor", "CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_COUNT", int + ) + kvikio_reactor_dispatch: kvikio.RemoteReactorDispatch | Unspecified = _opt( + "executor", + "CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_DISPATCH", + _parse_reactor_dispatch, + ) + kvikio_request_ceiling: int | Unspecified = _opt( + "executor", "CUDF_POLARS__EXECUTOR__KVIKIO_REQUEST_CEILING", int + ) max_concurrent_io_tasks: int | dict[str, int] | Unspecified | None = _opt( "executor", "CUDF_POLARS__EXECUTOR__MAX_CONCURRENT_IO_TASKS", diff --git a/python/cudf_polars/cudf_polars/engine/ray.py b/python/cudf_polars/cudf_polars/engine/ray.py index 3e7ffab3004d..1173d16f3aa5 100644 --- a/python/cudf_polars/cudf_polars/engine/ray.py +++ b/python/cudf_polars/cudf_polars/engine/ray.py @@ -55,8 +55,7 @@ MemoryResourceConfig, RayContext, configure_kvikio, - resolve_kvikio_nthreads, - resolve_kvikio_statistics, + resolve_kvikio_executor_options, ) if TYPE_CHECKING: @@ -253,8 +252,14 @@ def __init__( nranks: int, rapidsmpf_options_as_bytes: bytes, num_py_executors: int, - kvikio_nthreads: int, + kvikio_nthreads: int | None, kvikio_statistics: bool, + kvikio_remote_io_backend: kvikio.RemoteIOBackend, + kvikio_task_size: int, + kvikio_bounce_buffer_bytes: int, + kvikio_reactor_count: int, + kvikio_reactor_dispatch: kvikio.RemoteReactorDispatch, + kvikio_request_ceiling: int, hardware_binding: HardwareBindingPolicy, memory_resource_config: MemoryResourceConfig | None, worker_id: uuid.UUID, @@ -262,7 +267,15 @@ def __init__( quent_enabled: bool, ) -> None: bind_to_gpu(hardware_binding) - configure_kvikio(kvikio_nthreads) + configure_kvikio( + kvikio_nthreads, + remote_io_backend=kvikio_remote_io_backend, + task_size=kvikio_task_size, + bounce_buffer_bytes=kvikio_bounce_buffer_bytes, + reactor_count=kvikio_reactor_count, + reactor_dispatch=kvikio_reactor_dispatch, + request_ceiling=kvikio_request_ceiling, + ) memory_resource_config = ( memory_resource_config or MemoryResourceConfig.default() ) @@ -364,8 +377,14 @@ def reset( self, *, rapidsmpf_options_as_bytes: bytes, - kvikio_nthreads: int, + kvikio_nthreads: int | None, kvikio_statistics: bool, + kvikio_remote_io_backend: kvikio.RemoteIOBackend, + kvikio_task_size: int, + kvikio_bounce_buffer_bytes: int, + kvikio_reactor_count: int, + kvikio_reactor_dispatch: kvikio.RemoteReactorDispatch, + kvikio_request_ceiling: int, ) -> None: """ Rebuild the streaming Context with new options. @@ -379,12 +398,33 @@ def reset( Serialized :class:`Options` to install. kvikio_nthreads Number of kvikio threads to configure on this worker process. + ``None`` defers to kvikio's own built-in default. kvikio_statistics Whether to collect KvikIO I/O statistics on this rank. + kvikio_remote_io_backend + The kvikio remote I/O backend to configure on this worker process. + kvikio_task_size + Size, in bytes, of the kvikio task size to configure on this worker process. + kvikio_bounce_buffer_bytes + Size, in bytes, of the kvikio bounce buffer to configure on this worker process. + kvikio_reactor_count + Number of ``MULTI_POLL`` reactor threads to configure on this worker process. + kvikio_reactor_dispatch + ``MULTI_POLL`` reactor dispatch policy to configure on this worker process. + kvikio_request_ceiling + ``MULTI_POLL`` concurrent-request ceiling to configure on this worker process. """ if self._ctx is None: raise RuntimeError("reset() requires setup_worker() to have run") - configure_kvikio(kvikio_nthreads) + configure_kvikio( + kvikio_nthreads, + remote_io_backend=kvikio_remote_io_backend, + task_size=kvikio_task_size, + bounce_buffer_bytes=kvikio_bounce_buffer_bytes, + reactor_count=kvikio_reactor_count, + reactor_dispatch=kvikio_reactor_dispatch, + request_ceiling=kvikio_request_ceiling, + ) assert self._comm is not None # Collective: all ranks idle before any rank tears down its Context. if self._comm.nranks > 1: @@ -787,13 +827,7 @@ def __init__( ray_init_options: dict[str, Any] | None = None, num_ranks: int | None = None, ) -> None: - executor_options = executor_options or {} - executor_options.setdefault( - "kvikio_nthreads", resolve_kvikio_nthreads(executor_options) - ) - executor_options.setdefault( - "kvikio_statistics", resolve_kvikio_statistics(executor_options) - ) + executor_options = resolve_kvikio_executor_options(executor_options or {}) engine_options = engine_options or {} ray_init_options = ray_init_options or {} @@ -875,6 +909,16 @@ def __init__( ), kvikio_nthreads=executor_options["kvikio_nthreads"], kvikio_statistics=executor_options["kvikio_statistics"], + kvikio_remote_io_backend=executor_options[ + "kvikio_remote_io_backend" + ], + kvikio_task_size=executor_options["kvikio_task_size"], + kvikio_bounce_buffer_bytes=executor_options[ + "kvikio_bounce_buffer_bytes" + ], + kvikio_reactor_count=executor_options["kvikio_reactor_count"], + kvikio_reactor_dispatch=executor_options["kvikio_reactor_dispatch"], + kvikio_request_ceiling=executor_options["kvikio_request_ceiling"], hardware_binding=hw_binding, memory_resource_config=mr_config, worker_id=worker_id, @@ -927,16 +971,16 @@ def _reset( ) executor_options = executor_options or {} existing_executor_options = self.config.get("executor_options", {}) - if isinstance(existing_executor_options, dict): - existing_quent_context = existing_executor_options.get("quent_context") - if existing_quent_context is not None: - executor_options.setdefault("quent_context", existing_quent_context) - existing_kvikio_nthreads = existing_executor_options.get("kvikio_nthreads") - if existing_kvikio_nthreads is not None: - executor_options.setdefault("kvikio_nthreads", existing_kvikio_nthreads) - executor_options.setdefault( - "kvikio_statistics", resolve_kvikio_statistics(executor_options) - ) + if not isinstance(existing_executor_options, dict): + existing_executor_options = {} + existing_quent_context = existing_executor_options.get("quent_context") + if existing_quent_context is not None: + executor_options.setdefault("quent_context", existing_quent_context) + if "kvikio_nthreads" in existing_executor_options: + executor_options.setdefault( + "kvikio_nthreads", existing_executor_options["kvikio_nthreads"] + ) + executor_options = resolve_kvikio_executor_options(executor_options) engine_options = engine_options or {} self.rapidsmpf_options = resolve_rapidsmpf_options(rapidsmpf_options) rapidsmpf_options_as_bytes = self.rapidsmpf_options.serialize() @@ -950,6 +994,16 @@ def _reset( rapidsmpf_options_as_bytes=rapidsmpf_options_as_bytes, kvikio_nthreads=executor_options["kvikio_nthreads"], kvikio_statistics=executor_options["kvikio_statistics"], + kvikio_remote_io_backend=executor_options[ + "kvikio_remote_io_backend" + ], + kvikio_task_size=executor_options["kvikio_task_size"], + kvikio_bounce_buffer_bytes=executor_options[ + "kvikio_bounce_buffer_bytes" + ], + kvikio_reactor_count=executor_options["kvikio_reactor_count"], + kvikio_reactor_dispatch=executor_options["kvikio_reactor_dispatch"], + kvikio_request_ceiling=executor_options["kvikio_request_ceiling"], ) for rank in self._rank_actors ] @@ -1171,8 +1225,11 @@ def _run(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> list[T]: ).values() ) + # TODO: adopt polars' Engine.execute(lf, *, optimizations) contract + # (added in polars>=1.43) so we can return our own result type from + # LazyFrame.execute(engine=...) too (See https://github.com/NVIDIA/cudf/issues/22917). @unstable() - def execute(self, lf: pl.LazyFrame) -> PersistedQueryResult: + def execute(self, lf: pl.LazyFrame) -> PersistedQueryResult: # type: ignore[override] """ Execute a :class:`~polars.LazyFrame` and return a distributed result. diff --git a/python/cudf_polars/cudf_polars/engine/spmd.py b/python/cudf_polars/cudf_polars/engine/spmd.py index bc85d909f00e..59076cce245f 100644 --- a/python/cudf_polars/cudf_polars/engine/spmd.py +++ b/python/cudf_polars/cudf_polars/engine/spmd.py @@ -65,8 +65,7 @@ SPMDContext, StreamingExecutor, configure_kvikio, - resolve_kvikio_nthreads, - resolve_kvikio_statistics, + resolve_kvikio_executor_options, ) if TYPE_CHECKING: @@ -419,13 +418,7 @@ def __init__( executor_options: dict[str, Any] | None = None, engine_options: dict[str, Any] | None = None, ) -> None: - executor_options = executor_options or {} - executor_options.setdefault( - "kvikio_nthreads", resolve_kvikio_nthreads(executor_options) - ) - executor_options.setdefault( - "kvikio_statistics", resolve_kvikio_statistics(executor_options) - ) + executor_options = resolve_kvikio_executor_options(executor_options or {}) engine_options = engine_options or {} quent_context: cudf_polars.quent.QuentContext | None = executor_options.get( @@ -443,7 +436,15 @@ def __init__( ) bind_to_gpu(hw_binding) - configure_kvikio(executor_options["kvikio_nthreads"]) + configure_kvikio( + executor_options["kvikio_nthreads"], + remote_io_backend=executor_options["kvikio_remote_io_backend"], + task_size=executor_options["kvikio_task_size"], + bounce_buffer_bytes=executor_options["kvikio_bounce_buffer_bytes"], + reactor_count=executor_options["kvikio_reactor_count"], + reactor_dispatch=executor_options["kvikio_reactor_dispatch"], + request_ceiling=executor_options["kvikio_request_ceiling"], + ) self.rapidsmpf_options = resolve_rapidsmpf_options(rapidsmpf_options) mr_config: MemoryResourceConfig = engine_options.get( @@ -627,16 +628,24 @@ def _reset( ) executor_options = executor_options or {} existing_executor_options = self.config.get("executor_options", {}) - if isinstance(existing_executor_options, dict): - existing_quent_context = existing_executor_options.get("quent_context") - if existing_quent_context is not None: - executor_options.setdefault("quent_context", existing_quent_context) - existing_kvikio_nthreads = existing_executor_options.get("kvikio_nthreads") - if existing_kvikio_nthreads is not None: - executor_options.setdefault("kvikio_nthreads", existing_kvikio_nthreads) - configure_kvikio(executor_options["kvikio_nthreads"]) - executor_options.setdefault( - "kvikio_statistics", resolve_kvikio_statistics(executor_options) + if not isinstance(existing_executor_options, dict): + existing_executor_options = {} + existing_quent_context = existing_executor_options.get("quent_context") + if existing_quent_context is not None: + executor_options.setdefault("quent_context", existing_quent_context) + if "kvikio_nthreads" in existing_executor_options: + executor_options.setdefault( + "kvikio_nthreads", existing_executor_options["kvikio_nthreads"] + ) + executor_options = resolve_kvikio_executor_options(executor_options) + configure_kvikio( + executor_options["kvikio_nthreads"], + remote_io_backend=executor_options["kvikio_remote_io_backend"], + task_size=executor_options["kvikio_task_size"], + bounce_buffer_bytes=executor_options["kvikio_bounce_buffer_bytes"], + reactor_count=executor_options["kvikio_reactor_count"], + reactor_dispatch=executor_options["kvikio_reactor_dispatch"], + request_ceiling=executor_options["kvikio_request_ceiling"], ) engine_options = engine_options or {} quent_context: cudf_polars.quent.QuentContext | None = executor_options.get( @@ -893,8 +902,11 @@ def _run(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> list[T]: return [json.loads(r) for r in results] + # TODO: adopt polars' Engine.execute(lf, *, optimizations) contract + # (added in polars>=1.43) so we can return our own result type from + # LazyFrame.execute(engine=...) too (See https://github.com/NVIDIA/cudf/issues/22917). @unstable() - def execute(self, lf: pl.LazyFrame) -> PersistedQueryResult: + def execute(self, lf: pl.LazyFrame) -> PersistedQueryResult: # type: ignore[override] """ Execute a :class:`~polars.LazyFrame` and return a GPU-resident result. diff --git a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py index 6305ea206327..87938cf04c63 100644 --- a/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py +++ b/python/cudf_polars/cudf_polars/testing/inject_gpu_engine.py @@ -10,7 +10,6 @@ from functools import partialmethod from typing import TYPE_CHECKING -import numpy import packaging.version import pytest @@ -214,7 +213,6 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/io/test_csv.py::test_read_csv_only_loads_selected_columns": "Memory usage won't be correct due to GPU", "tests/unit/io/test_delta.py::test_scan_delta_version": "Need to expose hive partitioning", "tests/unit/io/test_delta.py::test_scan_delta_relative": "Need to expose hive partitioning", - "tests/unit/io/test_delta.py::test_read_delta_version": "Need to expose hive partitioning", "tests/unit/io/test_delta.py::test_scan_delta_schema_evolution_nested_struct_field_19915": "Need to expose hive partitioning", "tests/unit/io/test_delta.py::test_scan_delta_nanosecond_timestamp": "polars generates the wrong schema: https://github.com/pola-rs/polars/issues/23949", "tests/unit/io/test_delta.py::test_scan_delta_nanosecond_timestamp_nested": "polars generates the wrong schema: https://github.com/pola-rs/polars/issues/23949", @@ -223,8 +221,12 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/io/test_iceberg.py::test_scan_iceberg_extra_struct_fields": "Iceberg support not yet implemented in cudf-polars", "tests/unit/io/test_iceberg.py::test_scan_iceberg_column_deletion": "Iceberg schema evolution not yet implemented in cudf-polars", "tests/unit/io/test_iceberg.py::test_scan_iceberg_nested_column_cast_deletion_rename": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", - "tests/unit/io/test_iceberg.py::test_scan_iceberg_parquet_prefilter_with_column_mapping": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", + "tests/unit/io/test_iceberg.py::test_scan_iceberg_parquet_prefilter_with_column_mapping[True]": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", + "tests/unit/io/test_iceberg.py::test_scan_iceberg_parquet_prefilter_with_column_mapping[False]": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", "tests/unit/io/test_iceberg.py::test_fill_missing_fields_with_identity_partition_values_nested": "Iceberg partition column injection not yet implemented in cudf-polars", + "tests/unit/io/test_iceberg.py::test_sink_iceberg_schema_merge": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", + "tests/unit/io/test_iceberg.py::test_sink_iceberg_schema_merge_nested": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", + "tests/unit/io/test_iceberg.py::test_sink_iceberg_pickle": "Iceberg column_mapping (schema evolution) not yet implemented in cudf-polars", "tests/unit/io/test_iceberg.py::test_scan_iceberg_fast_count[native]": "Iceberg fast count from metadata not yet supported in cudf-polars", "tests/unit/io/test_iceberg.py::test_iceberg_filter_bool_26474": "Iceberg support not yet implemented in cudf-polars", "tests/unit/io/test_io_plugin.py::test_defer_validate_false": "cudf-polars always validates the IO source schema, so validate_schema=False dtype mismatches are unsupported on GPU", @@ -266,35 +268,34 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/io/test_scan.py::test_async_read_21945[scan_type2]": "chunked-reader + include_file_paths bug: chunk.num_rows_per_source only reflects the first chunk", "tests/unit/io/test_scan.py::test_async_read_21945[scan_type3]": "chunked-reader + include_file_paths bug: chunk.num_rows_per_source only reflects the first chunk", "tests/unit/io/test_sink.py::test_collect_all_lazy": "SinkMultiple not supported by InMemory CPU Engine, which we fallback to. See pola-rs/polars/pull/26537", + "tests/unit/io/test_sink.py::test_sink_metrics[True-parquet]": "Debug output on stderr doesn't match", + "tests/unit/io/test_sink.py::test_sink_metrics[True-ipc]": "Debug output on stderr doesn't match", + "tests/unit/io/test_sink.py::test_sink_metrics[True-csv]": "Debug output on stderr doesn't match", + "tests/unit/io/test_sink.py::test_sink_metrics[True-ndjson]": "Debug output on stderr doesn't match", + "tests/unit/io/test_sink.py::test_sink_metrics[False-parquet]": "Debug output on stderr doesn't match", + "tests/unit/io/test_sink.py::test_sink_metrics[False-ipc]": "Debug output on stderr doesn't match", + "tests/unit/io/test_sink.py::test_sink_metrics[False-csv]": "Debug output on stderr doesn't match", + "tests/unit/io/test_sink.py::test_sink_metrics[False-ndjson]": "Debug output on stderr doesn't match", "tests/unit/io/test_lazy_parquet.py::test_scan_parquet_ignores_dtype_mismatch_for_non_projected_columns_19249[False-False]": "Needs some variant of cudf#16394", "tests/unit/io/test_lazy_parquet.py::test_scan_parquet_ignores_dtype_mismatch_for_non_projected_columns_19249[True-False]": "Needs some variant of cudf#16394", "tests/unit/io/test_lazy_parquet.py::test_parquet_unaligned_schema_read[False]": "Incomplete handling of projected reads with mismatching schemas, cudf#16394", "tests/unit/io/test_parquet.py::test_read_parquet_only_loads_selected_columns_15098": "Memory usage won't be correct due to GPU", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-False-columns]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-False-columns]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-none]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-prefiltered]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-row_groups]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection0-True-columns]": "Mismatching column read cudf#16394", - "tests/unit/io/test_parquet.py::test_allow_missing_columns[projection1-True-columns]": "Mismatching column read cudf#16394", "tests/unit/io/test_parquet.py::test_scan_parquet_filter_statistics_load_missing_column_21391": "Mismatching column read cudf#16394", "tests/unit/io/test_parquet.py::test_binary_offset_roundtrip": "binary offset type unsupported", + "tests/unit/io/test_lazy_csv.py::test_scan_csv_schema_new_columns_dtypes[foods1.csv]": "CSV reader's use_cols_names picks the wrong columns when combined with a column_names_overwrite rename", + "tests/unit/io/test_lazy_csv.py::test_scan_csv_schema_new_columns_dtypes[foods*.csv]": "CSV reader's use_cols_names picks the wrong columns when combined with a column_names_overwrite rename", "tests/unit/lazyframe/test_engine_selection.py::test_engine_import_error_raises[gpu]": "Expect this to pass because cudf-polars is installed", "tests/unit/lazyframe/test_engine_selection.py::test_engine_import_error_raises[engine1]": "Expect this to pass because cudf-polars is installed", + "tests/unit/lazyframe/test_engine.py::test_object_engine_affinity_drives_collect": "This plugin forces engine= on LazyFrame.collect via partialmethod, so Config.set_engine_affinity's custom engine is never reached", + "tests/unit/lazyframe/test_query_monitoring.py::test_config_enable_monitoring": "This plugin forces engine= on LazyFrame.collect via partialmethod, so Config.enable_monitoring's streaming-affinity/observer hooks don't see the expected call counts", + "tests/unit/lazyframe/test_query_monitoring.py::test_config_scope_monitoring": "This plugin forces engine= on LazyFrame.collect via partialmethod, so Config.enable_monitoring's streaming-affinity/observer hooks don't see the expected call counts", + "tests/unit/lazyframe/test_query_monitoring.py::test_engine_affinity_object_carries_monitoring": "This plugin forces engine= on LazyFrame.collect via partialmethod, so Config.engine_affinity's streaming-affinity/observer hooks don't see the expected call counts", + "tests/unit/lazyframe/test_projections.py::test_projection_pushdown_union_len_pushdown_28657[concat]": "https://github.com/NVIDIA/cudf/issues/24112", + "tests/unit/lazyframe/test_projections.py::test_projection_pushdown_union_len_pushdown_28657[union]": "https://github.com/NVIDIA/cudf/issues/24112", "tests/unit/lazyframe/test_lazyframe.py::test_round[dtype2-123.55-1-123.6]": "libcudf HALF_EVEN rounding bug for Float64 with decimal_places > 0. See https://github.com/NVIDIA/cudf/issues/21319", "tests/unit/lazyframe/test_lazyframe.py::test_cast_frame": "Casting that raises not supported on GPU", "tests/unit/lazyframe/test_lazyframe.py::test_lazy_cache_hit": "Debug output on stderr doesn't match", "tests/unit/operations/aggregation/test_aggregations.py::test_binary_op_agg_context_no_simplify_expr_12423": "groupby-agg of just literals should not produce collect_list", - "tests/unit/operations/aggregation/test_aggregations.py::test_nan_inf_aggregation": "treatment of nans and nulls together is different in libcudf and polars in groupby-agg context", "tests/unit/operations/test_abs.py::test_abs_duration": "Need to raise for unsupported uops on timelike values", "tests/unit/operations/test_group_by.py::test_group_by_mean_by_dtype[input10-expected10-Date-output_dtype10]": "Unsupported groupby-agg for a particular dtype", "tests/unit/operations/test_group_by.py::test_group_by_mean_by_dtype[input11-expected11-input_dtype11-output_dtype11]": "Unsupported groupby-agg for a particular dtype", @@ -305,23 +306,19 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/operations/test_group_by.py::test_group_by_median_by_dtype[input16-expected16-input_dtype16-output_dtype16]": "Unsupported groupby-agg for a particular dtype", "tests/unit/operations/test_group_by.py::test_grouped_slice_literals[False]": "List literal loses nesting in groupby-agg: cudf#19610", "tests/unit/operations/test_group_by.py::test_grouped_slice_literals[True]": "List literal loses nesting in groupby-agg: cudf#19610", - "tests/unit/operations/test_group_by.py::test_group_broadcast_binary_apply_expr_25046[pow-rhs1-lhs0]": "libcudf integer pow uses float exp/log, so 3**1 rounds to 2", - "tests/unit/operations/test_group_by.py::test_group_broadcast_binary_apply_expr_25046[pow-rhs2-lhs0]": "libcudf integer pow uses float exp/log, so 3**1 rounds to 2", - "tests/unit/operations/test_group_by.py::test_group_broadcast_binary_apply_expr_25046[pow-rhs3-lhs0]": "libcudf integer pow uses float exp/log, so 3**1 rounds to 2", - "tests/unit/operations/test_group_by.py::test_group_broadcast_binary_apply_expr_25046[pow-rhs4-lhs0]": "libcudf integer pow uses float exp/log, so 3**1 rounds to 2", - "tests/unit/operations/test_group_by.py::test_group_by_binary_agg_with_literal": "Incorrect broadcasting of literals in groupby-agg", "tests/unit/operations/test_group_by.py::test_group_by_lit_series": "Incorrect broadcasting of literals in groupby-agg", - "tests/unit/operations/test_group_by.py::test_group_by_series_lit_22103[False]": "Incorrect broadcasting of literals in groupby-agg", - "tests/unit/operations/test_group_by.py::test_group_by_series_lit_22103[True]": "Incorrect broadcasting of literals in groupby-agg", "tests/unit/operations/test_join.py::test_cross_join_slice_pushdown": "Need to implement slice pushdown for cross joins", # TODO: As of polars 1.34, the column names for left and right came in unaligned, which causes the dtypes to mismatch when calling plc.replace.replace_nulls # Need to investigate what changed in polars "tests/unit/operations/test_join.py::test_join_coalesce_column_order_23177": "Misaligned left/right column names left and right tables in join op", + "tests/unit/operations/test_inequality_join.py::test_join_where_how_left": "join_where(how='left'/'right') is translated through the equi-join path using the predicate's columns as literal equality keys, ignoring the actual inequality condition", + "tests/unit/operations/test_inequality_join.py::test_join_where_how_right": "join_where(how='left'/'right') is translated through the equi-join path using the predicate's columns as literal equality keys, ignoring the actual inequality condition", + "tests/unit/operations/test_inequality_join.py::test_join_where_how_left_external_filter_not_folded_into_on_nested_loop": "join_where(how='left'/'right') is translated through the equi-join path using the predicate's columns as literal equality keys, ignoring the actual inequality condition", + "tests/unit/sql/test_joins.py::test_non_equi_left_join": "SQL LEFT/RIGHT JOIN with an inequality condition is translated through the equi-join path, ignoring the inequality condition", + "tests/unit/sql/test_joins.py::test_non_equi_left_join_null_keys": "SQL LEFT/RIGHT JOIN with an inequality condition is translated through the equi-join path, ignoring the inequality condition", + "tests/unit/sql/test_joins.py::test_non_equi_right_join_pure": "SQL LEFT/RIGHT JOIN with an inequality condition is translated through the equi-join path, ignoring the inequality condition", + "tests/unit/sql/test_joins.py::test_non_equi_outer_joins_unsupported[RIGHT]": "SQL LEFT/RIGHT JOIN with an inequality condition is translated through the equi-join path, ignoring the inequality condition", "tests/unit/operations/namespaces/string/test_pad.py::test_str_zfill_unicode_not_respected": "polars doesn't add zeros for unicode characters.", - "tests/unit/sql/test_cast.py::test_cast_errors[values0-values::uint8-conversion from `f64` to `u64` failed]": "Casting that raises not supported on GPU", - "tests/unit/sql/test_cast.py::test_cast_errors[values1-values::uint4-conversion from `i64` to `u32` failed]": "Casting that raises not supported on GPU", - "tests/unit/sql/test_cast.py::test_cast_errors[values2-values::int1-conversion from `i64` to `i8` failed]": "Casting that raises not supported on GPU", - "tests/unit/sql/test_cast.py::test_cast_errors[values5-values::int4-conversion from `str` to `i32` failed]": "Cast raises, but error user receives is wrong", "tests/unit/lazyframe/test_predicates.py::test_predicate_pushdown_split_pushable": "Casting that raises not supported on GPU", "tests/unit/lazyframe/test_predicates.py::test_filter_contradiction_fallible_error_handling": "Casting that raises not supported on GPU", "tests/unit/sql/test_miscellaneous.py::test_read_csv": "Incorrect handling of missing_is_null in read_csv", @@ -329,16 +326,7 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/lazyframe/test_cse.py::test_cse_predicate_self_join[True]": "Debug output on stderr doesn't match, see https://github.com/NVIDIA/cudf/issues/22967", "tests/unit/io/test_scan_row_deletion.py::test_scan_row_deletion_skips_file_with_all_rows_deleted": "The test intentionally corrupts the parquet file, so we cannot read the row count from the header.", "tests/unit/io/test_multiscan.py::test_multiscan_row_index[scan_csv-write_csv-csv]": "Debug output on stderr doesn't match", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-columns]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-row_groups]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-prefiltered]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[True-none]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-columns]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-row_groups]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-prefiltered]": "allow_missing_columns argument in read_parquet not translated in IR", - "tests/unit/io/test_lazy_parquet.py::test_parquet_schema_arg[False-none]": "allow_missing_columns argument in read_parquet not translated in IR", "tests/unit/io/test_multiscan.py::test_multiscan_row_index[scan_csv-write_csv]": "CSV multiscan with row_index and no row limit is not yet supported.", - "tests/unit/operations/namespaces/test_binary.py::test_binary_compounded_literal_aggstate_24460": "List literal loses nesting in gather: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_gather_get_on_literal_24101[lit1-0-False]": "List literal loses nesting in gather: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_gather_get_on_literal_24101[lit1-idx1-False]": "List literal loses nesting in gather: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_gather_get_on_literal_24101[lit1-idx2-False]": "List literal loses nesting in gather: cudf#19610", @@ -349,16 +337,10 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-0-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/operations/test_slice.py::test_schema_slice_on_literal_23999[lit1-offset1-len1-False]": "List literal loses nesting in slice: cudf#19610", "tests/unit/functions/test_concat.py::test_concat_with_empty_dataframes_strict_25725": "https://github.com/NVIDIA/cudf/issues/21644", - "tests/unit/sql/test_window_functions.py::test_over_with_cumulative_window_funcs": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", - "tests/unit/sql/test_window_functions.py::test_window_cumulative_agg_with_nulls": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", - "tests/unit/sql/test_window_functions.py::test_window_named_window": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", "tests/unit/operations/test_window.py::test_over_literal_cum_sum_26800": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", "tests/unit/operations/namespaces/array/test_array.py::test_array_idx_size_limit_eval": "polars-internal IdxSize chunking debug assertion does not apply with the GPU engine", - "tests/unit/operations/aggregation/test_aggregations.py::test_implode_and_agg": "implode + agg returns a mismatched dtype", - "tests/unit/operations/aggregation/test_aggregations.py::test_duration_aggs": "Unsupported libcudf reduction operator for Duration dtype", - "tests/unit/operations/aggregation/test_aggregations.py::test_boolean_aggs": "boolean-agg mean floating-point precision mismatch", "tests/unit/io/test_scan.py::test_scan_sink_metrics_multiple_phases": "sink metrics are not reported by the GPU engine", - "tests/unit/io/test_parquet.py::test_read_parquet_legacy_nested_maps_27159": "legacy nested-map parquet read produces a mismatched result", + "tests/unit/io/test_parquet.py::test_read_parquet_concatenated_gzip_members_28787": "Non-deterministic garbage in the last row; likely a libcudf GZIP multi-member decompression bug", "tests/unit/datatypes/test_struct.py::test_struct_equal_missing_null_25360": "struct equality with a null raises libcudf 'Index out of bounds' (get_element)", "tests/unit/datatypes/test_temporal.py::test_tz_aware_truncate": "truncate/round operate on the UTC instant instead of the zone's local wall-clock time", } @@ -366,6 +348,13 @@ def pytest_report_header(config: pytest.Config) -> str: TESTS_TO_SKIP: dict[str, str] = { "tests/unit/operations/test_profile.py::test_profile_with_cse": "Shape assertion won't match", + # CI has shown two different failure modes across runs (a PanicException on + # an invalid datetime, and garbage/uninitialized-looking datetime values), + # suggesting a real non-deterministic bug in CSV datetime fallback parsing + # under ignore_errors=True. Locally this hangs indefinitely instead of + # reproducing either failure, so it can't be xfailed; skip instead. + "tests/unit/io/test_csv.py::test_csv_datetime_fallback_contract[chunk-size-default]": "Non-deterministic CSV datetime fallback parsing; hangs locally instead of reproducing", + "tests/unit/io/test_csv.py::test_csv_datetime_fallback_contract[chunk-size-7]": "Non-deterministic CSV datetime fallback parsing; hangs locally instead of reproducing", # value_counts / struct-expansion row ordering is not guaranteed, so the GPU # result may or may not match CPU. Skip rather than xfail to avoid a flaky # XPASS/FAIL (these pass on some runs and fail on others). @@ -415,9 +404,11 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/io/test_scan.py::test_scan_metrics[True-parquet]": "Checks to IO metric logs specific to Polars CPU", "tests/unit/io/test_scan.py::test_scan_metrics[True-csv]": "Checks to IO metric logs specific to Polars CPU", "tests/unit/io/test_scan.py::test_scan_metrics[True-ndjson]": "Checks to IO metric logs specific to Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[True-ipc]": "Checks to IO metric logs specific to Polars CPU", "tests/unit/io/test_scan.py::test_scan_metrics[False-parquet]": "Checks to IO metric logs specific to Polars CPU", "tests/unit/io/test_scan.py::test_scan_metrics[False-csv]": "Checks to IO metric logs specific to Polars CPU", "tests/unit/io/test_scan.py::test_scan_metrics[False-ndjson]": "Checks to IO metric logs specific to Polars CPU", + "tests/unit/io/test_scan.py::test_scan_metrics[False-ipc]": "Checks to IO metric logs specific to Polars CPU", # polars 1.42 updated these tests to also assert deprecated_call and strict=True ShapeError # in the same test function. The SPMD engine fails on the strict=True collect() inside a # pytest.raises block because the DeprecationWarning from how='horizontal' propagates differently @@ -427,15 +418,6 @@ def pytest_report_header(config: pytest.Config) -> str: } -if packaging.version.parse(numpy.__version__) >= packaging.version.parse("2.5.0"): - # TODO: remove once cudf-polars supports polars==1.44 - EXPECTED_FAILURES.update( - { - "tests/unit/constructors/test_series.py::test_series_init_np_temporal_with_nat_15518": "DeprecationWarning from Numpy: https://github.com/pola-rs/polars/pull/28782", - } - ) - - if packaging.version.parse(sqlite3.sqlite_version) <= packaging.version.parse("3.44.0"): # These tests rely on features not available in older versions of sqlite. TESTS_TO_SKIP.update( @@ -465,6 +447,11 @@ def pytest_report_header(config: pytest.Config) -> str: # 1) Tests that are too slow with --inject-gpu-engine-blocksize=small due to many small partitions for large data STREAMING_ENGINE_TESTS_TO_SKIP: Mapping[str, str] = { "tests/unit/operations/aggregation/test_aggregations.py::test_boolean_aggs": "float difference in std/var in the unit of least precision", + # Crashes the worker instead of raising cleanly; skip rather than xfail + # since a crashed worker never reports a result. See + # https://github.com/NVIDIA/cudf/issues/24112 + "tests/unit/lazyframe/test_projections.py::test_projection_pushdown_union_len_pushdown_28657[concat]": "Materializing a huge virtual literal frame crashes the worker under the streaming engine", + "tests/unit/lazyframe/test_projections.py::test_projection_pushdown_union_len_pushdown_28657[union]": "Materializing a huge virtual literal frame crashes the worker under the streaming engine", # No deterministic key sort (https://github.com/NVIDIA/cudf/issues/21641): # passes on some streaming runs and fails on others, so skip rather than # xfail to avoid a flaky XPASS/FAIL. @@ -539,8 +526,6 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/functions/test_when_then.py::test_mismatched_height_should_raise[ternary_expr1-df1]": "Correct polars.exceptions.ShapeError raised but it's in a ExceptionGroup", "tests/unit/operations/test_slice.py::test_slice_pushdown_literal_projection_14349": "https://github.com/NVIDIA/cudf/issues/22072", "tests/unit/operations/test_group_by.py::test_group_by_lit_series": "Incorrect broadcasting of literals in groupby-agg", - "tests/unit/operations/test_group_by.py::test_group_by_series_partitioned": "https://github.com/NVIDIA/cudf/issues/22072", - "tests/unit/operations/test_group_by.py::test_partitioned_group_by_chunked": "https://github.com/NVIDIA/cudf/issues/22072", "tests/unit/operations/test_group_by.py::test_unique_head_tail_26429[1]": "https://github.com/NVIDIA/cudf/issues/22075", "tests/unit/operations/test_group_by.py::test_unique_head_tail_26429[4]": "https://github.com/NVIDIA/cudf/issues/22075", "tests/unit/operations/aggregation/test_aggregations.py::test_item_too_many": "Correct polars.exceptions.ComputeError raised but it's in an ExceptionGroup", @@ -548,9 +533,6 @@ def pytest_report_header(config: pytest.Config) -> str: "tests/unit/operations/test_join.py::test_empty_outer_join_22206": "https://github.com/NVIDIA/cudf/issues/22084", "tests/unit/operations/test_replace.py::test_replace_invalid_old_dtype": "Correct InvalidOperationError raised but it's in an ExceptionGroup", "tests/unit/operations/test_window.py::test_over_literal_cum_sum_26800": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", - "tests/unit/sql/test_joins.py::test_cross_join_unnest_from_cte": "https://github.com/NVIDIA/cudf/issues/22073", - "tests/unit/sql/test_window_functions.py::test_over_with_cumulative_window_funcs": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", - "tests/unit/sql/test_window_functions.py::test_window_cumulative_agg_with_nulls": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", "tests/unit/sql/test_window_functions.py::test_window_multiple_named_window": "TODO: https://github.com/NVIDIA/cudf/pull/22048#discussion_r3238041970", "tests/unit/functions/test_concat.py::test_concat_horizontal_lazy_strict_raises_shape_error_27415": "horizontal-concat strict height-mismatch raised inside an ExceptionGroup under the streaming engine", "tests/unit/io/test_io_plugin.py::test_defer_validate_true": "correct SchemaError raised but wrapped in an ExceptionGroup under the streaming engine", diff --git a/python/cudf_polars/cudf_polars/utils/config.py b/python/cudf_polars/cudf_polars/utils/config.py index 37844aea6d10..098511f0cc3d 100644 --- a/python/cudf_polars/cudf_polars/utils/config.py +++ b/python/cudf_polars/cudf_polars/utils/config.py @@ -255,33 +255,282 @@ def resolve_kvikio_statistics(executor_options: dict[str, Any]) -> bool: return value if isinstance(value, bool) else _bool_converter(value) -def resolve_kvikio_nthreads(executor_options: dict[str, Any]) -> int: - """Resolve kvikio thread count from executor options with env var fallback.""" +def resolve_kvikio_nthreads( + executor_options: dict[str, Any], + *, + remote_io_backend: kvikio.RemoteIOBackend | None = None, +) -> int | None: + """ + Resolve kvikio thread count from executor options with env var fallback. + + Defaults to 256 for the ``EASY_THREADPOOL`` backend (tuned for cloud + object-store IO). Under ``MULTI_POLL``, this pool is only used for local + (non-remote) I/O, so unless explicitly overridden, resolution returns + ``None`` to defer to kvikio's own built-in default (the ``KVIKIO_NTHREADS`` + environment variable, else 4). ``remote_io_backend`` should be the + already-resolved backend (e.g. via :func:`resolve_kvikio_remote_io_backend`); + if omitted, it is resolved from ``executor_options`` with the same env var + fallback. + """ + value = executor_options.get( + "kvikio_nthreads", os.environ.get("CUDF_POLARS__EXECUTOR__KVIKIO_NTHREADS") + ) + if value is not None: + return int(value) + if remote_io_backend is None: + remote_io_backend = resolve_kvikio_remote_io_backend(executor_options) + if remote_io_backend == kvikio.RemoteIOBackend.MULTI_POLL: + return None + return int(os.environ.get("KVIKIO_NTHREADS", "256")) + + +def resolve_kvikio_remote_io_backend( + executor_options: dict[str, Any], +) -> kvikio.RemoteIOBackend: + """Resolve the kvikio remote I/O backend from executor options with env var fallback.""" + value = executor_options.get( + "kvikio_remote_io_backend", + os.environ.get( + "CUDF_POLARS__EXECUTOR__KVIKIO_REMOTE_IO_BACKEND", + os.environ.get("KVIKIO_REMOTE_IO_BACKEND", "MULTI_POLL"), + ), + ) + if isinstance(value, kvikio.RemoteIOBackend): + return value + return kvikio.RemoteIOBackend[str(value).upper()] + + +def resolve_kvikio_bounce_buffer_bytes(executor_options: dict[str, Any]) -> int: + """ + Resolve the kvikio bounce buffer size, in bytes, with env var fallback. + + Unlike ``kvikio_task_size``, this setting is not specific to the + ``MULTI_POLL`` backend: it sizes the host-memory staging buffer used for + any device-memory transfer (local file I/O, mmap I/O, and remote reads + under both ``MULTI_POLL`` and ``EASY_THREADPOOL``), so it applies + regardless of the active remote I/O backend. + """ + return int( + executor_options.get( + "kvikio_bounce_buffer_bytes", + os.environ.get( + "CUDF_POLARS__EXECUTOR__KVIKIO_BOUNCE_BUFFER_BYTES", + os.environ.get("KVIKIO_BOUNCE_BUFFER_SIZE", str(16 * 1024 * 1024)), + ), + ) + ) + + +def resolve_kvikio_task_size( + executor_options: dict[str, Any], + *, + remote_io_backend: kvikio.RemoteIOBackend | None = None, +) -> int: + """ + Resolve the kvikio task size, in bytes, with env var fallback. + + Defaults to 16 MiB for the ``MULTI_POLL`` remote I/O backend and 64 MiB for + ``EASY_THREADPOOL``, unless overridden via ``executor_options`` or an + environment variable. ``remote_io_backend`` should be the already-resolved + backend (e.g. via :func:`resolve_kvikio_remote_io_backend`); if omitted, it + is resolved from ``executor_options`` with the same env var fallback. + """ + if remote_io_backend is None: + remote_io_backend = resolve_kvikio_remote_io_backend(executor_options) + default = ( + 16 * 1024 * 1024 + if remote_io_backend == kvikio.RemoteIOBackend.MULTI_POLL + else 64 * 1024 * 1024 + ) + return int( + executor_options.get( + "kvikio_task_size", + os.environ.get( + "CUDF_POLARS__EXECUTOR__KVIKIO_TASK_SIZE", + os.environ.get("KVIKIO_TASK_SIZE", str(default)), + ), + ) + ) + + +def resolve_kvikio_reactor_count(executor_options: dict[str, Any]) -> int: + """Resolve the number of MULTI_POLL reactor threads, with env var fallback.""" + return int( + executor_options.get( + "kvikio_reactor_count", + os.environ.get( + "CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_COUNT", + os.environ.get("KVIKIO_REMOTE_IO_NUM_REACTORS", "24"), + ), + ) + ) + + +def resolve_kvikio_reactor_dispatch( + executor_options: dict[str, Any], +) -> kvikio.RemoteReactorDispatch: + """Resolve the MULTI_POLL reactor dispatch policy, with env var fallback.""" + value = executor_options.get( + "kvikio_reactor_dispatch", + os.environ.get( + "CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_DISPATCH", + os.environ.get("KVIKIO_REMOTE_IO_REACTOR_DISPATCH", "PER_CHUNK"), + ), + ) + if isinstance(value, kvikio.RemoteReactorDispatch): + return value + return kvikio.RemoteReactorDispatch[str(value).upper()] + + +def resolve_kvikio_request_ceiling(executor_options: dict[str, Any]) -> int: + """Resolve the MULTI_POLL concurrent-request ceiling, with env var fallback.""" return int( executor_options.get( - "kvikio_nthreads", + "kvikio_request_ceiling", os.environ.get( - "CUDF_POLARS__EXECUTOR__KVIKIO_NTHREADS", - os.environ.get("KVIKIO_NTHREADS", "256"), + "CUDF_POLARS__EXECUTOR__KVIKIO_REQUEST_CEILING", + os.environ.get("KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS", "256"), ), ) ) -def configure_kvikio(nthreads: int) -> None: - """Set the remote I/O backend to ``EASY_THREADPOOL`` with ``nthreads`` threads.""" +def resolve_kvikio_executor_options(executor_options: dict[str, Any]) -> dict[str, Any]: + """ + Resolve every ``kvikio_*`` executor option in place, via ``setdefault``. + + Existing keys (already resolved, e.g. carried over across a ``_reset``) are + left untouched. ``kvikio_remote_io_backend`` is resolved first since + ``kvikio_nthreads`` and ``kvikio_task_size`` default differently depending + on it; the shared, single source of truth for that ordering lives here so + the dask/ray/spmd engines don't each re-implement it. + """ + executor_options.setdefault( + "kvikio_statistics", resolve_kvikio_statistics(executor_options) + ) + executor_options.setdefault( + "kvikio_remote_io_backend", + resolve_kvikio_remote_io_backend(executor_options), + ) + executor_options.setdefault( + "kvikio_nthreads", + resolve_kvikio_nthreads( + executor_options, + remote_io_backend=executor_options["kvikio_remote_io_backend"], + ), + ) + executor_options.setdefault( + "kvikio_task_size", + resolve_kvikio_task_size( + executor_options, + remote_io_backend=executor_options["kvikio_remote_io_backend"], + ), + ) + executor_options.setdefault( + "kvikio_bounce_buffer_bytes", + resolve_kvikio_bounce_buffer_bytes(executor_options), + ) + executor_options.setdefault( + "kvikio_reactor_count", resolve_kvikio_reactor_count(executor_options) + ) + executor_options.setdefault( + "kvikio_reactor_dispatch", + resolve_kvikio_reactor_dispatch(executor_options), + ) + executor_options.setdefault( + "kvikio_request_ceiling", + resolve_kvikio_request_ceiling(executor_options), + ) + return executor_options + + +# kvikio.defaults property names that configure_kvikio may pass to +# kvikio.defaults.set (checked at the end of configure_kvikio). Also used by +# callers (e.g. tests that need to snapshot/restore kvikio's process-global +# defaults) so they don't have to duplicate this list by hand. +KVIKIO_CONFIGURABLE_PROPERTIES = ( + "remote_io_backend", + "task_size", + "bounce_buffer_size", + "remote_io_num_reactors", + "remote_io_reactor_dispatch", + "remote_io_max_concurrent_requests", + "num_threads", +) + +# Process-lifetime settings that govern kvikio's MULTI_POLL reactor pool. +# KvikIO fixes these once the pool starts (i.e. after the first MULTI_POLL +# remote I/O), so they must be configured through the individual-setter API. +# Re-applying an already-active value is skipped; a conflicting value is left +# for KvikIO to reject with its lifecycle error. +KVIKIO_REACTOR_POOL_PROPERTIES = frozenset( + { + "remote_io_num_reactors", + "remote_io_reactor_dispatch", + "remote_io_max_concurrent_requests", + } +) + + +def configure_kvikio( + nthreads: int | None, + *, + remote_io_backend: kvikio.RemoteIOBackend = kvikio.RemoteIOBackend.MULTI_POLL, + task_size: int | None = None, + bounce_buffer_bytes: int = 16 * 1024 * 1024, + reactor_count: int = 24, + reactor_dispatch: kvikio.RemoteReactorDispatch = kvikio.RemoteReactorDispatch.PER_CHUNK, + request_ceiling: int = 256, +) -> None: + """ + Configure kvikio for cudf-polars I/O. + + ``nthreads=None`` defers the size of kvikio's local-I/O thread pool to its + own built-in default (the ``KVIKIO_NTHREADS`` environment variable, else 4). + This is the typical case under the ``MULTI_POLL`` remote I/O backend, whose + remote I/O does not use this pool at all. + """ # HACK: libcudf calls set_up_kvikio() on the first IO op and that resets the thread # pool (default is 4 if KVIKIO_NTHREADS is unset), undoing anything we set via # kvikio.defaults. We call it here with our nthreads so later when it's called in # libcudf it's a no-op. The explicit kvikio.defaults.set below handles subsequent # calls to configure_kvikio (call_once only fires once). pylibcudf.utils._set_up_kvikio(nthreads) - kvikio.defaults.set( - { - "num_threads": nthreads, - "remote_io_backend": kvikio.RemoteIOBackend.EASY_THREADPOOL, + if task_size is None: + task_size = resolve_kvikio_task_size({}, remote_io_backend=remote_io_backend) + settings: dict[str, Any] = { + "remote_io_backend": remote_io_backend, + "task_size": task_size, + # Sizes the host staging buffer for device-memory transfers. This applies + # to local and remote I/O under both backends, so it is always set. + "bounce_buffer_size": bounce_buffer_bytes, + } + reactor_settings: dict[str, Any] = {} + if remote_io_backend == kvikio.RemoteIOBackend.MULTI_POLL: + # The MULTI_POLL backend ignores kvikio's EASY_THREADPOOL thread pool for + # remote I/O, so num_threads is not configured here. Local I/O still uses + # the thread pool created by `_set_up_kvikio` above. The reactor settings + # below are ignored by kvikio when EASY_THREADPOOL is active, so they are + # left unset in that case. + reactor_settings = { + "remote_io_num_reactors": reactor_count, + "remote_io_reactor_dispatch": reactor_dispatch, + "remote_io_max_concurrent_requests": request_ceiling, } - ) + else: + if nthreads is None: + nthreads = resolve_kvikio_nthreads({}, remote_io_backend=remote_io_backend) + assert nthreads is not None # EASY_THREADPOOL always resolves to an int + settings["num_threads"] = nthreads + assert set(settings).isdisjoint(KVIKIO_REACTOR_POOL_PROPERTIES) + assert set(settings) | set(reactor_settings) <= set(KVIKIO_CONFIGURABLE_PROPERTIES) + kvikio.defaults.set(settings) + # KvikIO does not permit reactor-pool settings in the dict form. Avoid a + # no-op individual setter after the pool has started, because KvikIO + # correctly rejects all setters at that point. + for key, value in reactor_settings.items(): + if kvikio.defaults.get(key) != value: + kvikio.defaults.set(key, value) def _bool_converter(v: str) -> bool: @@ -866,9 +1115,16 @@ class StreamingExecutor: Maximum number of workers for the Python ThreadPoolExecutor. Default is 8. kvikio_nthreads - Number of threads in the kvikio ``EASY_THREADPOOL`` thread pool. - Defaults to 256, which is tuned for cloud object-store IO. This can be - set via + Number of threads in kvikio's local-I/O thread pool (used for local + file I/O under both backends, and for remote I/O under + ``EASY_THREADPOOL``; ``MULTI_POLL`` remote I/O uses the reactor threads + instead, controlled separately by ``kvikio_reactor_count``). Defaults + to 256 (tuned for cloud object-store IO) when + ``kvikio_remote_io_backend`` is ``EASY_THREADPOOL``. Under + ``MULTI_POLL``, defaults to ``None``, which defers to kvikio's own + built-in default (the ``KVIKIO_NTHREADS`` environment variable, else 4) + rather than spinning up a 256-thread pool that backend would rarely + use. This can be set via - ``executor_options`` passed to ``polars.GPUEngine`` - the ``CUDF_POLARS__EXECUTOR__KVIKIO_NTHREADS`` environment variable @@ -878,8 +1134,9 @@ class StreamingExecutor: kvikio uses a single process-wide thread pool. When a streaming engine is created, it configures that pool to ``kvikio_nthreads`` - threads. This operation blocks until all in-flight kvikio IO in the - process completes and then rebuilds the pool. As a result: + threads (skipped when ``kvikio_nthreads`` is ``None``). This + operation blocks until all in-flight kvikio IO in the process + completes and then rebuilds the pool. As a result: - Any code in the same process that is using kvikio concurrently at engine creation time will be disrupted. @@ -887,6 +1144,59 @@ class StreamingExecutor: engine creation will be overridden. Use the ``kvikio_nthreads`` executor option or ``KVIKIO_NTHREADS`` environment variable instead. + kvikio_remote_io_backend + The kvikio remote I/O backend. ``kvikio.RemoteIOBackend.MULTI_POLL`` by + default. This can be set via + + - ``executor_options`` passed to ``polars.GPUEngine`` + - the ``CUDF_POLARS__EXECUTOR__KVIKIO_REMOTE_IO_BACKEND`` environment variable + - the ``KVIKIO_REMOTE_IO_BACKEND`` environment variable (lower precedence) + kvikio_task_size + Size, in bytes, of the chunks kvikio splits reads into for parallel + dispatch. Defaults to 16 MiB for the ``MULTI_POLL`` backend and 64 MiB + for ``EASY_THREADPOOL``. Applies to local and remote I/O under both + backends. This can be set via + + - ``executor_options`` passed to ``polars.GPUEngine`` + - the ``CUDF_POLARS__EXECUTOR__KVIKIO_TASK_SIZE`` environment variable + - the ``KVIKIO_TASK_SIZE`` environment variable (lower precedence) + kvikio_bounce_buffer_bytes + Size, in bytes, of the kvikio bounce buffer used to stage host memory + for device-memory transfers. Defaults to 16 MiB. Applies to local and + remote I/O under both backends (not specific to ``MULTI_POLL``), and + under ``MULTI_POLL`` must be at least ``kvikio_task_size`` for + device-buffer reads. This can be set via + + - ``executor_options`` passed to ``polars.GPUEngine`` + - the ``CUDF_POLARS__EXECUTOR__KVIKIO_BOUNCE_BUFFER_BYTES`` environment variable + - the ``KVIKIO_BOUNCE_BUFFER_SIZE`` environment variable (lower precedence) + kvikio_reactor_count + Number of reactor threads used by the ``MULTI_POLL`` remote I/O backend. + Ignored when ``kvikio_remote_io_backend`` is not ``MULTI_POLL``. Defaults + to 24. This can be set via + + - ``executor_options`` passed to ``polars.GPUEngine`` + - the ``CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_COUNT`` environment variable + - the ``KVIKIO_REMOTE_IO_NUM_REACTORS`` environment variable (lower precedence) + kvikio_reactor_dispatch + How sub-ranges of one read are distributed across reactor threads under + the ``MULTI_POLL`` remote I/O backend. Ignored when + ``kvikio_remote_io_backend`` is not ``MULTI_POLL``. Defaults to + ``kvikio.RemoteReactorDispatch.PER_CHUNK``. This can be set via + + - ``executor_options`` passed to ``polars.GPUEngine`` + - the ``CUDF_POLARS__EXECUTOR__KVIKIO_REACTOR_DISPATCH`` environment variable + - the ``KVIKIO_REMOTE_IO_REACTOR_DISPATCH`` environment variable (lower precedence) + kvikio_request_ceiling + Maximum number of concurrent in-flight requests across all reactor + threads under the ``MULTI_POLL`` remote I/O backend. 0 means unlimited. + Ignored when ``kvikio_remote_io_backend`` is not ``MULTI_POLL``. Defaults + to 256. This can be set via + + - ``executor_options`` passed to ``polars.GPUEngine`` + - the ``CUDF_POLARS__EXECUTOR__KVIKIO_REQUEST_CEILING`` environment variable + - the ``KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS`` environment variable + (lower precedence) quent_context Quent tracing context. When ``None`` (default), Quent tracing is disabled. Pass a :class:`~cudf_polars.quent.QuentContext` instance to enable tracing. @@ -962,14 +1272,35 @@ class StreamingExecutor: f"{_env_prefix}__NUM_PY_EXECUTORS", int, default=8 ) ) - kvikio_nthreads: int = dataclasses.field( - default_factory=lambda: resolve_kvikio_nthreads({}) - ) + # `None` is resolved in `__post_init__`, once `kvikio_remote_io_backend` (an + # explicit constructor argument, if any) is known, since the default value + # depends on the backend. The resolved value may itself remain `None` + # (deferring to kvikio's own built-in default) under `MULTI_POLL`. + kvikio_nthreads: int | None = None kvikio_statistics: bool = dataclasses.field( default_factory=_make_default_factory( f"{_env_prefix}__KVIKIO_STATISTICS", _bool_converter, default=False ) ) + kvikio_remote_io_backend: kvikio.RemoteIOBackend = dataclasses.field( + default_factory=lambda: resolve_kvikio_remote_io_backend({}) + ) + # `None` is resolved in `__post_init__`, once `kvikio_remote_io_backend` (an + # explicit constructor argument, if any) is known, since the default value + # depends on the backend. + kvikio_task_size: int | None = None + kvikio_bounce_buffer_bytes: int = dataclasses.field( + default_factory=lambda: resolve_kvikio_bounce_buffer_bytes({}) + ) + kvikio_reactor_count: int = dataclasses.field( + default_factory=lambda: resolve_kvikio_reactor_count({}) + ) + kvikio_reactor_dispatch: kvikio.RemoteReactorDispatch = dataclasses.field( + default_factory=lambda: resolve_kvikio_reactor_dispatch({}) + ) + kvikio_request_ceiling: int = dataclasses.field( + default_factory=lambda: resolve_kvikio_request_ceiling({}) + ) min_device_size: int | None = None spmd_context: SPMDContext | None = None @@ -1051,10 +1382,65 @@ def __post_init__(self) -> None: # noqa: D105 raise TypeError("client_device_threshold must be a float") if not isinstance(self.num_py_executors, int): raise TypeError("num_py_executors must be an int") - if not isinstance(self.kvikio_nthreads, int): - raise TypeError("kvikio_nthreads must be an int") - if self.kvikio_nthreads <= 0: - raise ValueError("kvikio_nthreads must be positive") + if not isinstance(self.kvikio_remote_io_backend, kvikio.RemoteIOBackend): + object.__setattr__( + self, + "kvikio_remote_io_backend", + kvikio.RemoteIOBackend[str(self.kvikio_remote_io_backend).upper()], + ) + if self.kvikio_nthreads is None: + # May remain `None` here, deferring to kvikio's own built-in + # default; see `resolve_kvikio_nthreads`. + object.__setattr__( + self, + "kvikio_nthreads", + resolve_kvikio_nthreads( + {}, remote_io_backend=self.kvikio_remote_io_backend + ), + ) + if self.kvikio_nthreads is not None: + if not isinstance(self.kvikio_nthreads, int): + raise TypeError("kvikio_nthreads must be an int or None") + if self.kvikio_nthreads <= 0: + raise ValueError("kvikio_nthreads must be positive") + if self.kvikio_task_size is None: + object.__setattr__( + self, + "kvikio_task_size", + resolve_kvikio_task_size( + {}, remote_io_backend=self.kvikio_remote_io_backend + ), + ) + if not isinstance(self.kvikio_task_size, int): + raise TypeError("kvikio_task_size must be an int") + if self.kvikio_task_size <= 0: + raise ValueError("kvikio_task_size must be positive") + if not isinstance(self.kvikio_bounce_buffer_bytes, int): + raise TypeError("kvikio_bounce_buffer_bytes must be an int") + if self.kvikio_bounce_buffer_bytes <= 0: + raise ValueError("kvikio_bounce_buffer_bytes must be positive") + if ( + self.kvikio_remote_io_backend == kvikio.RemoteIOBackend.MULTI_POLL + and self.kvikio_bounce_buffer_bytes < self.kvikio_task_size + ): + raise ValueError( + "kvikio_bounce_buffer_bytes must be at least kvikio_task_size " + "for the MULTI_POLL backend" + ) + if not isinstance(self.kvikio_reactor_count, int): + raise TypeError("kvikio_reactor_count must be an int") + if self.kvikio_reactor_count <= 0: + raise ValueError("kvikio_reactor_count must be positive") + if not isinstance(self.kvikio_reactor_dispatch, kvikio.RemoteReactorDispatch): + object.__setattr__( + self, + "kvikio_reactor_dispatch", + kvikio.RemoteReactorDispatch[str(self.kvikio_reactor_dispatch).upper()], + ) + if not isinstance(self.kvikio_request_ceiling, int): + raise TypeError("kvikio_request_ceiling must be an int") + if self.kvikio_request_ceiling < 0: + raise ValueError("kvikio_request_ceiling must be non-negative") def __hash__(self) -> int: # noqa: D105 # dynamic_planning factory, a dataclass, isn't natively hashable. We'll dump it diff --git a/python/cudf_polars/cudf_polars/utils/dtypes.py b/python/cudf_polars/cudf_polars/utils/dtypes.py index 73c82e20c625..1693a79992e8 100644 --- a/python/cudf_polars/cudf_polars/utils/dtypes.py +++ b/python/cudf_polars/cudf_polars/utils/dtypes.py @@ -101,7 +101,7 @@ def can_cast(from_: plc.DataType, to: plc.DataType) -> bool: or ( to.id() == plc.TypeId.STRING and not from_is_empty - and is_numeric_not_bool(from_) + and (is_numeric_not_bool(from_) or plc.traits.is_boolean(from_)) ) or ( plc.traits.is_integral_not_bool(from_) diff --git a/python/cudf_polars/cudf_polars/utils/versions.py b/python/cudf_polars/cudf_polars/utils/versions.py index 3c74ef2cb298..4977e92756be 100644 --- a/python/cudf_polars/cudf_polars/utils/versions.py +++ b/python/cudf_polars/cudf_polars/utils/versions.py @@ -19,6 +19,8 @@ POLARS_VERSION_LT_140 = POLARS_VERSION < parse("1.40.0") POLARS_VERSION_LT_141 = POLARS_VERSION < parse("1.41.0") POLARS_VERSION_LT_142 = POLARS_VERSION < parse("1.42.0") +POLARS_VERSION_LT_143 = POLARS_VERSION < parse("1.43.0") +POLARS_VERSION_LT_144 = POLARS_VERSION < parse("1.44.0") def _ensure_polars_version() -> None: diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index cb32bbe44fd2..1b6cfdaef476 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "kvikio==26.10.*,>=0.0.0a0", "nvidia-ml-py>=12", "packaging", - "polars>=1.35,<1.43", + "polars>=1.35,<1.45", "pylibcudf==26.10.*,>=0.0.0a0", "rapidsmpf==26.10.*,>=0.0.0a0", "typing_extensions>=4.0.0", diff --git a/python/cudf_polars/tests/dsl/test_serialization.py b/python/cudf_polars/tests/dsl/test_serialization.py index 2cad17324682..970b1f4af39e 100644 --- a/python/cudf_polars/tests/dsl/test_serialization.py +++ b/python/cudf_polars/tests/dsl/test_serialization.py @@ -17,6 +17,8 @@ POLARS_VERSION_LT_138, POLARS_VERSION_LT_141, POLARS_VERSION_LT_142, + POLARS_VERSION_LT_143, + POLARS_VERSION_LT_144, ) @@ -61,6 +63,17 @@ def test_from_polars_all_names(function): if POLARS_VERSION_LT_142 and function == BooleanFunction: # 'IsSorted' was added to polars' BooleanFunction in 1.42. cudf_polars_names_set = cudf_polars_names_set - {"IsSorted"} + if POLARS_VERSION_LT_143 and function == StringFunction: + # 'ExtractMany', 'FindMany', and 'Format' were added to polars' + # StringFunction in 1.43. + cudf_polars_names_set = cudf_polars_names_set - { + "ExtractMany", + "FindMany", + "Format", + } + if POLARS_VERSION_LT_144 and function == StructFunction: + # 'DropFields' was added to polars' StructFunction in 1.44. + cudf_polars_names_set = cudf_polars_names_set - {"DropFields"} assert polars_names_set == cudf_polars_names_set names = function.Name if function == StructFunction: @@ -77,6 +90,14 @@ def test_from_polars_all_names(function): } if POLARS_VERSION_LT_142 and function == BooleanFunction: names = set(names) - {BooleanFunction.Name.IsSorted} + if POLARS_VERSION_LT_143 and function == StringFunction: + names = set(names) - { + StringFunction.Name.ExtractMany, + StringFunction.Name.FindMany, + StringFunction.Name.Format, + } + if POLARS_VERSION_LT_144 and function == StructFunction: + names = set(names) - {StructFunction.Name.DropFields} for name in names: attr = getattr(polars_function, name.name) assert function.Name.from_polars(attr) == name diff --git a/python/cudf_polars/tests/expressions/test_booleanfunction.py b/python/cudf_polars/tests/expressions/test_booleanfunction.py index 0084a7ab8ccc..bba573436f4b 100644 --- a/python/cudf_polars/tests/expressions/test_booleanfunction.py +++ b/python/cudf_polars/tests/expressions/test_booleanfunction.py @@ -265,6 +265,12 @@ def test_expr_is_in_empty_list(engine: pl.GPUEngine): assert_gpu_result_equal(q, engine=engine) +def test_expr_is_in_empty_list_with_nulls(engine: pl.GPUEngine): + ldf = pl.LazyFrame({"a": [1, None, 3]}) + q = ldf.select(pl.col("a").is_in([])) + assert_gpu_result_equal(q, engine=engine) + + @pytest.mark.parametrize( "needles,haystack", [ diff --git a/python/cudf_polars/tests/expressions/test_casting.py b/python/cudf_polars/tests/expressions/test_casting.py index 7df638e75b27..7633a2423ff2 100644 --- a/python/cudf_polars/tests/expressions/test_casting.py +++ b/python/cudf_polars/tests/expressions/test_casting.py @@ -92,10 +92,10 @@ def test_cast_from_string_unsupported(engine: pl.GPUEngine): assert_ir_translation_raises(query, engine, NotImplementedError) -def test_cast_to_string_unsupported(engine: pl.GPUEngine): - df = pl.LazyFrame({"a": [True]}) +def test_cast_bool_to_string(engine: pl.GPUEngine): + df = pl.LazyFrame({"a": [True, False, None]}) query = df.select(pl.col("a").cast(pl.String())) - assert_ir_translation_raises(query, engine, NotImplementedError) + assert_gpu_result_equal(query, engine=engine) def test_float_to_decimal_rounding(engine: pl.GPUEngine): diff --git a/python/cudf_polars/tests/expressions/test_len.py b/python/cudf_polars/tests/expressions/test_len.py index d2d29818bf12..f6f8ba4303db 100644 --- a/python/cudf_polars/tests/expressions/test_len.py +++ b/python/cudf_polars/tests/expressions/test_len.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -39,3 +39,13 @@ def test_col_len(engine: pl.GPUEngine, data): pl.col("b").len().alias("l3"), ) assert_gpu_result_equal(q, engine=engine) + + +def test_concat_len(engine: pl.GPUEngine): + # polars rewrites concat(...).select(len()) into + # col("len").cast(UInt128).sum().cast(IDX_DTYPE); see + # cudf_polars.dsl.translate._is_len_sum_uint128_node. + df1 = pl.LazyFrame({"a": [1, 2, 3]}) + df2 = pl.LazyFrame({"a": [4, 5, 6, 7]}) + q = pl.concat([df1, df2]).select(pl.len()) + assert_gpu_result_equal(q, engine=engine) diff --git a/python/cudf_polars/tests/expressions/test_numeric_unaryops.py b/python/cudf_polars/tests/expressions/test_numeric_unaryops.py index 1860f7690ba3..f9ea07f34e76 100644 --- a/python/cudf_polars/tests/expressions/test_numeric_unaryops.py +++ b/python/cudf_polars/tests/expressions/test_numeric_unaryops.py @@ -253,3 +253,15 @@ def test_atan2_unsupported(engine: pl.GPUEngine) -> None: df = pl.LazyFrame({"y": [1.0, 2.0, 3.0], "x": [4.0, 5.0, 6.0]}) q = df.select(pl.arctan2("y", "x")) assert_ir_translation_raises(q, engine, NotImplementedError) + + +@pytest.mark.parametrize("op", ["sqrt", "cbrt", "sin", "degrees", "pct_change"]) +def test_math_op_on_struct_unsupported(engine: pl.GPUEngine, op: str) -> None: + # https://github.com/pola-rs/polars/issues/28563: polars doesn't reject all + # of these at the schema level, so libcudf would otherwise crash on the + # struct input instead of cleanly falling back to CPU. + df = pl.LazyFrame( + {"a": pl.Series([{"x": 1}, {"x": 2}], dtype=pl.Struct({"x": pl.Int64}))} + ) + q = df.select(getattr(pl.col("a"), op)()) + assert_ir_translation_raises(q, engine, NotImplementedError) diff --git a/python/cudf_polars/tests/test_config.py b/python/cudf_polars/tests/test_config.py index bd4c338b13c3..ceaac67aa16e 100644 --- a/python/cudf_polars/tests/test_config.py +++ b/python/cudf_polars/tests/test_config.py @@ -3,9 +3,12 @@ from __future__ import annotations +import contextlib import dataclasses -from typing import cast +from typing import Any, cast +import kvikio +import kvikio.defaults import pytest import polars as pl @@ -28,6 +31,8 @@ assert_ir_translation_raises, ) from cudf_polars.utils.config import ( + KVIKIO_CONFIGURABLE_PROPERTIES, + KVIKIO_REACTOR_POOL_PROPERTIES, Cluster, ConfigOptions, DynamicPlanningOptions, @@ -38,6 +43,11 @@ ParquetOptions, StreamingExecutor, configure_kvikio, + resolve_kvikio_bounce_buffer_bytes, + resolve_kvikio_nthreads, + resolve_kvikio_reactor_dispatch, + resolve_kvikio_remote_io_backend, + resolve_kvikio_task_size, ) from cudf_polars.utils.cuda_stream import get_cuda_stream @@ -352,6 +362,72 @@ def test_kvikio_nthreads_non_positive_raises() -> None: ) +def test_kvikio_resolvers_accept_enum_values( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("KVIKIO_REMOTE_IO_BACKEND", "EASY_THREADPOOL") + monkeypatch.setenv("KVIKIO_NTHREADS", "32") + monkeypatch.delenv("CUDF_POLARS__EXECUTOR__KVIKIO_TASK_SIZE", raising=False) + monkeypatch.delenv("KVIKIO_TASK_SIZE", raising=False) + assert ( + resolve_kvikio_remote_io_backend({}) == kvikio.RemoteIOBackend.EASY_THREADPOOL + ) + assert ( + resolve_kvikio_remote_io_backend( + {"kvikio_remote_io_backend": kvikio.RemoteIOBackend.MULTI_POLL} + ) + == kvikio.RemoteIOBackend.MULTI_POLL + ) + assert resolve_kvikio_nthreads({}) == 32 + assert resolve_kvikio_task_size({}) == 64 * 1024 * 1024 + assert ( + resolve_kvikio_reactor_dispatch( + {"kvikio_reactor_dispatch": kvikio.RemoteReactorDispatch.PER_CHUNK} + ) + == kvikio.RemoteReactorDispatch.PER_CHUNK + ) + + +@pytest.mark.parametrize( + "option, value, match", + [ + ("kvikio_task_size", object(), "must be an int"), + ("kvikio_task_size", 0, "must be positive"), + ("kvikio_bounce_buffer_bytes", object(), "must be an int"), + ("kvikio_bounce_buffer_bytes", 0, "must be positive"), + ("kvikio_reactor_count", object(), "must be an int"), + ("kvikio_reactor_count", 0, "must be positive"), + ("kvikio_request_ceiling", object(), "must be an int"), + ("kvikio_request_ceiling", -1, "must be non-negative"), + ], +) +def test_validate_kvikio_options(option: str, value: object, match: str) -> None: + with pytest.raises((TypeError, ValueError), match=match): + StreamingExecutor( + cluster=Cluster.DEFAULT_SINGLETON, + **{option: cast("Any", value)}, + ) + + +def test_kvikio_bounce_buffer_must_cover_task_size() -> None: + with pytest.raises(ValueError, match="must be at least kvikio_task_size"): + StreamingExecutor( + cluster=Cluster.DEFAULT_SINGLETON, + kvikio_task_size=16, + kvikio_bounce_buffer_bytes=15, + ) + + +def test_streaming_executor_normalizes_kvikio_enum_strings() -> None: + executor = StreamingExecutor( + cluster=Cluster.DEFAULT_SINGLETON, + kvikio_remote_io_backend="EASY_THREADPOOL", + kvikio_reactor_dispatch="PER_CHUNK", + ) + assert executor.kvikio_remote_io_backend == kvikio.RemoteIOBackend.EASY_THREADPOOL + assert executor.kvikio_reactor_dispatch == kvikio.RemoteReactorDispatch.PER_CHUNK + + def test_executor_from_env(monkeypatch: pytest.MonkeyPatch) -> None: with monkeypatch.context() as m: m.setenv("CUDF_POLARS__EXECUTOR", "in-memory") @@ -1046,6 +1122,30 @@ def test_kvikio_nthreads_default(monkeypatch: pytest.MonkeyPatch) -> None: m.delenv("CUDF_POLARS__EXECUTOR__KVIKIO_NTHREADS", raising=False) m.delenv("KVIKIO_NTHREADS", raising=False) config = ConfigOptions.from_polars_engine(pl.GPUEngine(executor="streaming")) + assert config.executor.kvikio_nthreads is None + + with monkeypatch.context() as m: + m.delenv("CUDF_POLARS__EXECUTOR__KVIKIO_NTHREADS", raising=False) + m.delenv("KVIKIO_NTHREADS", raising=False) + m.setenv("KVIKIO_REMOTE_IO_BACKEND", "EASY_THREADPOOL") + config = ConfigOptions.from_polars_engine(pl.GPUEngine(executor="streaming")) + assert config.executor.kvikio_nthreads == 256 + + +def test_kvikio_nthreads_default_easy_threadpool( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with monkeypatch.context() as m: + m.delenv("CUDF_POLARS__EXECUTOR__KVIKIO_NTHREADS", raising=False) + m.delenv("KVIKIO_NTHREADS", raising=False) + config = ConfigOptions.from_polars_engine( + pl.GPUEngine( + executor="streaming", + executor_options={ + "kvikio_remote_io_backend": kvikio.RemoteIOBackend.EASY_THREADPOOL + }, + ) + ) assert config.executor.kvikio_nthreads == 256 @@ -1071,10 +1171,22 @@ def test_kvikio_nthreads_from_env( def test_kvikio_nthreads_from_kvikio_env( monkeypatch: pytest.MonkeyPatch, ) -> None: + # Under MULTI_POLL, KVIKIO_NTHREADS is still honored, but via kvikio's own + # deferred default rather than cudf-polars resolving it to a concrete int. with monkeypatch.context() as m: m.delenv("CUDF_POLARS__EXECUTOR__KVIKIO_NTHREADS", raising=False) m.setenv("KVIKIO_NTHREADS", "32") config = ConfigOptions.from_polars_engine(pl.GPUEngine(executor="streaming")) + assert config.executor.kvikio_nthreads is None + + config = ConfigOptions.from_polars_engine( + pl.GPUEngine( + executor="streaming", + executor_options={ + "kvikio_remote_io_backend": kvikio.RemoteIOBackend.EASY_THREADPOOL + }, + ) + ) assert config.executor.kvikio_nthreads == 32 @@ -1088,14 +1200,45 @@ def test_kvikio_nthreads_cudf_polars_env_takes_precedence( assert config.executor.kvikio_nthreads == 64 +@pytest.fixture +def kvikio_defaults_guard(): + """Snapshot and restore kvikio.defaults around a test. + + ``configure_kvikio`` mutates process-global kvikio defaults via + ``kvikio.defaults.set``. Without restoring them, a test that calls + ``configure_kvikio`` can leak settings into later tests. The properties + to snapshot come from ``KVIKIO_CONFIGURABLE_PROPERTIES``, the list + ``configure_kvikio`` itself draws from, so this fixture stays in sync + with configure_kvikio without duplicating its property names. + + ``KVIKIO_REACTOR_POOL_PROPERTIES`` are restored separately and best-effort: + kvikio permanently fixes them once the MULTI_POLL reactor pool has + started (e.g. via another test's real remote I/O), and raises if asked to + set them afterward even to their current value, so there is nothing to + revert in that case. + """ + original = {key: kvikio.defaults.get(key) for key in KVIKIO_CONFIGURABLE_PROPERTIES} + yield + kvikio.defaults.set( + { + key: value + for key, value in original.items() + if key not in KVIKIO_REACTOR_POOL_PROPERTIES + } + ) + # If another test already started the MULTI_POLL reactor pool, kvikio pins + # these properties for good and this reset is a no-op that raises. + for key in KVIKIO_REACTOR_POOL_PROPERTIES: + with contextlib.suppress(RuntimeError): + kvikio.defaults.set(key, original[key]) + + def test_configure_kvikio_sets_backend_and_threads( monkeypatch: pytest.MonkeyPatch, + kvikio_defaults_guard: None, ) -> None: - import kvikio - import kvikio.defaults - monkeypatch.delenv("KVIKIO_NTHREADS", raising=False) - configure_kvikio(42) + configure_kvikio(42, remote_io_backend=kvikio.RemoteIOBackend.EASY_THREADPOOL) assert kvikio.defaults.get("num_threads") == 42 assert ( kvikio.defaults.get("remote_io_backend") @@ -1103,6 +1246,152 @@ def test_configure_kvikio_sets_backend_and_threads( ) +def test_configure_kvikio_multi_poll_defaults( + monkeypatch: pytest.MonkeyPatch, + kvikio_defaults_guard: None, +) -> None: + monkeypatch.delenv("KVIKIO_NTHREADS", raising=False) + calls = [] + original_set = kvikio.defaults.set + original_reactor_settings = { + key: kvikio.defaults.get(key) for key in KVIKIO_REACTOR_POOL_PROPERTIES + } + + def record_set(*args): + calls.append(args) + return original_set(*args) + + monkeypatch.setattr(kvikio.defaults, "set", record_set) + configure_kvikio(42) + reactor_settings = { + "remote_io_num_reactors": 24, + "remote_io_reactor_dispatch": kvikio.RemoteReactorDispatch.PER_CHUNK, + "remote_io_max_concurrent_requests": 256, + } + assert all( + not ( + len(args) == 1 + and isinstance(args[0], dict) + and KVIKIO_REACTOR_POOL_PROPERTIES.intersection(args[0]) + ) + for args in calls + ) + for key, value in reactor_settings.items(): + if original_reactor_settings[key] != value: + assert (key, value) in calls + assert kvikio.defaults.get("remote_io_backend") == kvikio.RemoteIOBackend.MULTI_POLL + assert kvikio.defaults.get("remote_io_num_reactors") == 24 + assert ( + kvikio.defaults.get("remote_io_reactor_dispatch") + == kvikio.RemoteReactorDispatch.PER_CHUNK + ) + assert kvikio.defaults.get("remote_io_max_concurrent_requests") == 256 + assert kvikio.defaults.get("bounce_buffer_size") == 16 * 1024 * 1024 + assert kvikio.defaults.get("task_size") == 16 * 1024 * 1024 + + +def test_configure_kvikio_multi_poll_does_not_reset_reactor_settings( + monkeypatch: pytest.MonkeyPatch, + kvikio_defaults_guard: None, +) -> None: + configure_kvikio(42) + + calls = [] + original_set = kvikio.defaults.set + + def record_set(*args): + calls.append(args) + return original_set(*args) + + monkeypatch.setattr(kvikio.defaults, "set", record_set) + configure_kvikio(42) + + assert all( + not (len(args) == 2 and args[0] in KVIKIO_REACTOR_POOL_PROPERTIES) + for args in calls + ) + + +def test_configure_kvikio_easy_threadpool_task_size_default( + monkeypatch: pytest.MonkeyPatch, + kvikio_defaults_guard: None, +) -> None: + monkeypatch.delenv("KVIKIO_NTHREADS", raising=False) + monkeypatch.delenv("KVIKIO_TASK_SIZE", raising=False) + configure_kvikio(42, remote_io_backend=kvikio.RemoteIOBackend.EASY_THREADPOOL) + assert kvikio.defaults.get("task_size") == 64 * 1024 * 1024 + assert kvikio.defaults.get("num_threads") == 42 + + +def test_configure_kvikio_easy_threadpool_resolves_default_threads( + monkeypatch: pytest.MonkeyPatch, + kvikio_defaults_guard: None, +) -> None: + monkeypatch.delenv("KVIKIO_NTHREADS", raising=False) + monkeypatch.setattr( + cudf_polars.utils.config.pylibcudf.utils, "_set_up_kvikio", lambda _: None + ) + configure_kvikio(None, remote_io_backend=kvikio.RemoteIOBackend.EASY_THREADPOOL) + assert kvikio.defaults.get("num_threads") == 256 + + +def test_resolve_kvikio_bounce_buffer_bytes_backend_independent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("KVIKIO_BOUNCE_BUFFER_SIZE", raising=False) + monkeypatch.delenv( + "CUDF_POLARS__EXECUTOR__KVIKIO_BOUNCE_BUFFER_BYTES", raising=False + ) + + # Not backend-specific: the default is the same regardless of backend. + assert resolve_kvikio_bounce_buffer_bytes({}) == 16 * 1024 * 1024 + assert ( + resolve_kvikio_bounce_buffer_bytes({"kvikio_bounce_buffer_bytes": 123}) == 123 + ) + + +def test_resolve_kvikio_task_size_defaults_by_backend( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("KVIKIO_TASK_SIZE", raising=False) + monkeypatch.delenv("CUDF_POLARS__EXECUTOR__KVIKIO_TASK_SIZE", raising=False) + + assert ( + resolve_kvikio_task_size( + {}, remote_io_backend=kvikio.RemoteIOBackend.MULTI_POLL + ) + == 16 * 1024 * 1024 + ) + assert ( + resolve_kvikio_task_size( + {}, remote_io_backend=kvikio.RemoteIOBackend.EASY_THREADPOOL + ) + == 64 * 1024 * 1024 + ) + # An explicit executor_options override wins regardless of backend. + assert ( + resolve_kvikio_task_size( + {"kvikio_task_size": 123}, + remote_io_backend=kvikio.RemoteIOBackend.EASY_THREADPOOL, + ) + == 123 + ) + + +def test_streaming_executor_kvikio_task_size_follows_explicit_backend() -> None: + easy = StreamingExecutor( + cluster=Cluster.DEFAULT_SINGLETON, + kvikio_remote_io_backend=kvikio.RemoteIOBackend.EASY_THREADPOOL, + ) + assert easy.kvikio_task_size == 64 * 1024 * 1024 + + multi = StreamingExecutor( + cluster=Cluster.DEFAULT_SINGLETON, + kvikio_remote_io_backend=kvikio.RemoteIOBackend.MULTI_POLL, + ) + assert multi.kvikio_task_size == 16 * 1024 * 1024 + + def test_dask_sink_to_directory_false_raises() -> None: with pytest.raises( ValueError, match="The dask cluster requires sink_to_directory=True" diff --git a/python/cudf_polars/tests/test_merge_sorted.py b/python/cudf_polars/tests/test_merge_sorted.py index fc9bf72ba92e..6e6ec08f986b 100644 --- a/python/cudf_polars/tests/test_merge_sorted.py +++ b/python/cudf_polars/tests/test_merge_sorted.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -6,8 +6,12 @@ import polars as pl -from cudf_polars.testing.asserts import assert_gpu_result_equal +from cudf_polars.testing.asserts import ( + assert_gpu_result_equal, + assert_ir_translation_raises, +) from cudf_polars.testing.engine_utils import is_streaming_engine +from cudf_polars.utils.versions import POLARS_VERSION_LT_143 @pytest.mark.parametrize("descending", [True, False]) @@ -32,6 +36,24 @@ def test_merge_sorted_without_nulls(engine: pl.GPUEngine, descending, request): assert_gpu_result_equal(q, engine=engine) +@pytest.mark.skipif( + POLARS_VERSION_LT_143, reason="Merging on multiple keys added in polars 1.43" +) +def test_merge_sorted_multiple_keys_not_supported(engine: pl.GPUEngine): + df0 = pl.LazyFrame( + {"name": ["steve", "elise", "bob"], "age": [42, 44, 18], "height": [5, 6, 5]} + ).sort(["age", "height"]) + df1 = pl.LazyFrame( + { + "name": ["anna", "megan", "steve", "thomas"], + "age": [21, 33, 42, 20], + "height": [5, 5, 5, 5], + } + ).sort(["age", "height"]) + q = df0.merge_sorted(df1, key=["age", "height"]) + assert_ir_translation_raises(q, engine, NotImplementedError) + + @pytest.mark.parametrize( "descending", [ diff --git a/python/cudf_streaming/CMakeLists.txt b/python/cudf_streaming/CMakeLists.txt index b30856cc16c9..bb1dcf94fa4b 100644 --- a/python/cudf_streaming/CMakeLists.txt +++ b/python/cudf_streaming/CMakeLists.txt @@ -36,11 +36,6 @@ find_package(CUDAToolkit REQUIRED) include(rapids-cython-core) rapids_cython_init() -# Enable Cython's translation cache when CI supplies a persistent cache directory. -if(DEFINED ENV{CYTHON_CACHE_DIR} AND NOT "$ENV{CYTHON_CACHE_DIR}" STREQUAL "") - list(PREPEND CYTHON_FLAGS "--cache") -endif() - set(cython_sources cudf_streaming/approx_distinct_count.pyx cudf_streaming/bloom_filter.pyx diff --git a/python/pylibcudf/CMakeLists.txt b/python/pylibcudf/CMakeLists.txt index 5dd033751f81..dec65c4299dc 100644 --- a/python/pylibcudf/CMakeLists.txt +++ b/python/pylibcudf/CMakeLists.txt @@ -36,9 +36,4 @@ include(rapids-cython-core) rapids_cython_init() -# Enable Cython's translation cache when CI supplies a persistent cache directory. -if(DEFINED ENV{CYTHON_CACHE_DIR} AND NOT "$ENV{CYTHON_CACHE_DIR}" STREQUAL "") - list(PREPEND CYTHON_FLAGS "--cache") -endif() - add_subdirectory(pylibcudf) diff --git a/skills/accelerated-computing-cudf/BENCHMARK.md b/skills/accelerated-computing-cudf/BENCHMARK.md index ee2cd251651b..81c752aad877 100644 --- a/skills/accelerated-computing-cudf/BENCHMARK.md +++ b/skills/accelerated-computing-cudf/BENCHMARK.md @@ -1,35 +1,23 @@ # Skill Benchmark: accelerated-computing-cudf -> ✅ **Overall verdict: PASS — Recommended for publication** +> ⚠️ **Overall verdict: INCOMPLETE — Required evidence is missing** -## Publication Recommendation - -Recommended for publication based on the completed evaluation evidence in this report. +One or more required evaluation tiers did not complete, so this benchmark is not publication-complete. ## Evaluation Metadata - Skill: `accelerated-computing-cudf` -- Evaluation date: 2026-08-27 -- Evaluator version: `1.3.2` +- Evaluation date: 2026-09-11 +- Evaluator version: `1.5.6` - Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) - Tasks: 13 evaluation tasks (12 positive, 1 negative) -- Dataset digest: `sha256:7d24a0888ebeec0a84f5675ca556ed370e2b7a7f22c249d18c4c4f85a769bc52` (skill-evaluator-dataset-snapshot/1) -- Attempts per task: 1 -- Environment: `local` +- Dataset digest: `sha256:307ff81fa3f0d04ee89889dcedc5ac0208fc0eb5dba112e1c8ed515a08fa3ba9` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 3 +- Environment: `k8s-sandbox` +- Tier 2 evidence: required for publication - Tier 3 evidence: required for publication -Tasks ran on the trusted local host; local mode is not sandboxed. - -## Execution and Provenance - -- Validation status: `passed` -- Report generation: `complete` -- Evaluator version: `1.3.2` -- Git commit: `0117bc2e3e54da4244a656466526c5b1b5a559ea` -- Content type: requested `auto`, detected `skill` -- Container image: `gitlab-master.nvidia.com:5005/nvcarps/ci-group/nvcarps-ci/skillevaluator-ci:sha-0117bc2e3e54da4244a656466526c5b1b5a559ea` -- Container image digest: `not recorded` -- Tier 3: requested `true`, executed `true`, status `succeeded` +Each task attempt ran in its own isolated sandbox pod. ## What This Report Answers @@ -45,16 +33,56 @@ The three-tier evaluation checks whether the skill: | Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 74% → 88% (+14 points) | 68% → 83% (+15 points) | -| Security | 85% → 69% (-15 points) | 54% → 46% (-8 points) | -| Correctness | 100% → 100% (±0 points) | 100% → 98% (-2 points) | -| Discoverability | 47% → 89% (+42 points) | 45% → 85% (+40 points) | -| Effectiveness | 96% → 96% (±0 points) | 95% → 94% (-1 points) | -| Efficiency | 44% → 87% (+43 points) | 46% → 89% (+43 points) | - -**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. - -Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. +| Overall | Not available | 84.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | Not available | 76.9% → 69.2% (-7.7 points) | +| Correctness | Not available | 100.0% → 100.0% (±0.0 points) | +| Discoverability | Not available | 81.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | Not available | 94.4% → 90.9% (-3.5 points) | +| Efficiency | Not available | 82.3% — baseline ran, but no comparable score was available; uplift unavailable | + +**How to read this table:** baseline is the same task attempted without the target skill. Scores are rounded to one decimal; threshold-adjacent values use additional precision so their displayed band matches the verdict. Uplift is derived from those displayed scores and shown in percentage points. + +Example: `47.0% → 92.0% (+45.0 points)` means the skill-assisted run scored 92.0%, 45.0 percentage points above its 47.0% no-skill baseline. + +A partial dimension was calculated from only the available configured signals; review the detailed report before relying on it. + +## Token Usage + +Actual Tier 3 execution usage is reported for every observed agent/case pair and both conditions. + +| Agent | Dataset case | With skill | Without skill | Delta | Change | Coverage | +|---|---|---:|---:|---:|---:|---| +| claude-code | All cases | 14,012,571 | 10,980,314 | N/A | N/A | skill 13/14; base 13/39 | +| claude-code | cudf-apply-udf__generic | 1,615,916 | 2,921,909 | -1,305,993 | -44.70% | skill 1/1; base 1/1 | +| claude-code | cudf-csv-etl__generic | 640,979 | 540,152 | +100,827 | +18.67% | skill 1/1; base 1/1 | +| claude-code | cudf-groupby-agg__generic | 1,334,220 | 1,114,950 | +219,270 | +19.67% | skill 1/1; base 1/1 | +| claude-code | cudf-multi-join__generic | 938,388 | 609,377 | +329,011 | +53.99% | skill 1/1; base 1/1 | +| claude-code | cudf-native-stream-handoff-boundary__generic | 753,844 | 676,138 | +77,706 | +11.49% | skill 1/1; base 1/1 | +| claude-code | cudf-null-handling__generic | 573,498 | 698,891 | -125,393 | -17.94% | skill 1/1; base 1/1 | +| claude-code | cudf-parquet-io__generic | 795,742 | 632,812 | +162,930 | +25.75% | skill 1/1; base 1/1 | +| claude-code | cudf-pivot-melt__generic | 969,475 | 636,461 | +333,014 | +52.32% | skill 1/1; base 1/1 | +| claude-code | cudf-string-ops__generic | 934,970 | 723,865 | +211,105 | +29.16% | skill 1/1; base 1/1 | +| claude-code | cudf-timeseries-resample__generic | 585,864 | 624,629 | -38,765 | -6.21% | skill 1/1; base 1/1 | +| claude-code | cudf-window-functions__generic | 2,207,481 | 675,332 | N/A | N/A | skill 1/2; base 1/1 | +| claude-code | negative-deep-learning-training__generic | 458,761 | 642,097 | -183,336 | -28.55% | skill 1/1; base 1/1 | +| claude-code | source-cudf-null-fillna-semantics__generic | 2,203,433 | 483,701 | +1,719,732 | +355.54% | skill 1/1; base 1/1 | +| codex | All cases | 4,399,248 | 3,330,594 | +1,068,654 | +32.09% | skill 13/13; base 13/13 | +| codex | cudf-apply-udf__generic | 366,287 | 333,591 | +32,696 | +9.80% | skill 1/1; base 1/1 | +| codex | cudf-csv-etl__generic | 368,472 | 218,866 | +149,606 | +68.36% | skill 1/1; base 1/1 | +| codex | cudf-groupby-agg__generic | 282,617 | 251,646 | +30,971 | +12.31% | skill 1/1; base 1/1 | +| codex | cudf-multi-join__generic | 246,898 | 177,146 | +69,752 | +39.38% | skill 1/1; base 1/1 | +| codex | cudf-native-stream-handoff-boundary__generic | 322,482 | 324,176 | -1,694 | -0.52% | skill 1/1; base 1/1 | +| codex | cudf-null-handling__generic | 481,827 | 313,356 | +168,471 | +53.76% | skill 1/1; base 1/1 | +| codex | cudf-parquet-io__generic | 294,404 | 206,582 | +87,822 | +42.51% | skill 1/1; base 1/1 | +| codex | cudf-pivot-melt__generic | 269,590 | 280,046 | -10,456 | -3.73% | skill 1/1; base 1/1 | +| codex | cudf-string-ops__generic | 258,274 | 162,821 | +95,453 | +58.62% | skill 1/1; base 1/1 | +| codex | cudf-timeseries-resample__generic | 385,513 | 291,946 | +93,567 | +32.05% | skill 1/1; base 1/1 | +| codex | cudf-window-functions__generic | 444,592 | 281,185 | +163,407 | +58.11% | skill 1/1; base 1/1 | +| codex | negative-deep-learning-training__generic | 268,298 | 220,178 | +48,120 | +21.86% | skill 1/1; base 1/1 | +| codex | source-cudf-null-fillna-semantics__generic | 409,994 | 269,055 | +140,939 | +52.38% | skill 1/1; base 1/1 | +| ALL AGENTS | Dataset aggregate | 18,411,819 | 14,310,908 | N/A | N/A | skill 26/27; base 26/52 | + +Prompt tokens include cached reads, so total tokens are `prompt + completion` (cached is not added twice). The Efficiency score uses `(prompt - cached) + completion`. N/A means the relevant trajectory counters were not available; coverage is never estimated. ## Tier Status @@ -62,7 +90,7 @@ Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 |---|---|---|---| | Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 3 finding(s) | | Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | -| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 13 task(s) | +| Tier 3 | Live agent evaluation | **NEUTRAL** | 2 agent(s); 13 task(s) | ## Findings and Observations @@ -86,23 +114,24 @@ Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 | Correctness | Is the answer correct? | `accuracy` (100%) | | Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | | Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | -| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | +| Efficiency | Did it avoid wasted tool calls and token usage? | `skill_efficiency` (50%) + `token_efficiency` (50%) | - Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. - Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. - Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. - The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. - Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). -- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. +- Efficiency is 50% tool-call productivity (the backward-compatible `skill_efficiency` wire id) and 50% `token_efficiency`. Positive-case skill routing is scored under Discoverability, not Efficiency; a negative case without a routing target is N/A. N/A sources are omitted, remaining weights are renormalized, and the dimension is marked partial. Signals present in this run: - `security` (Security): unsafe operations, secret leakage, and unauthorized access. -- `skill_execution` (Skill Execution): whether the expected skill was found and executed. -- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `skill_execution` (Skill Execution): whether the expected skill was selected, decoys were avoided, and the workflow executed. +- `skill_efficiency` (Tool Productivity): tool-call productivity (legacy wire id; routing is scored under Discoverability). - `accuracy` (Accuracy): final-answer correctness against the reference answer. - `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. - `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. +- `token_efficiency` (Token Efficiency): actual uncached prompt plus completion usage (50% of Efficiency). diff --git a/skills/accelerated-computing-cudf/SKILL.md b/skills/accelerated-computing-cudf/SKILL.md index 26ddef528e09..dc2d100f4d49 100644 --- a/skills/accelerated-computing-cudf/SKILL.md +++ b/skills/accelerated-computing-cudf/SKILL.md @@ -197,7 +197,7 @@ implementation. See `references/api-patterns.md` for nullable dtype examples. Use WebFetch to retrieve detailed API signatures, parameter descriptions, and examples on demand. -- **cuDF Documentation:** https://docs.rapids.ai/api/cudf/stable/ -- **dask-cuDF API Reference:** https://docs.rapids.ai/api/dask-cudf/stable/api/ +- **cuDF Documentation:** https://docs.nvidia.com/cudf/ +- **dask-cuDF API Reference:** https://docs.nvidia.com/dask-cudf/ - **GitHub:** https://github.com/NVIDIA/cudf - **CHANGELOG:** https://github.com/NVIDIA/cudf/blob/main/CHANGELOG.md diff --git a/skills/accelerated-computing-cudf/evals/evals.json b/skills/accelerated-computing-cudf/evals/evals.json index c7494decabad..97f0e949b29a 100644 --- a/skills/accelerated-computing-cudf/evals/evals.json +++ b/skills/accelerated-computing-cudf/evals/evals.json @@ -9,7 +9,11 @@ "evals/files/cudf-apply-udf/code/udf_pipeline.py" ], "ground_truth": "A successful answer uses the provided cudf-apply-udf starter files, especially code/udf_pipeline.py, to migrate the pandas DataFrame workload to cuDF where supported. It replaces row-wise apply/applymap or column UDF logic with vectorized cuDF expressions, Numba-compatible GPU logic, or a narrow compatibility boundary, preserves representative pandas results, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-apply-udf starter workspace and edits code/udf_pipeline.py rather than giving generic advice.", + "Replaces pandas row-wise apply/applymap or column UDF logic with vectorized cuDF expressions, Numba-compatible GPU logic, or a clearly documented CPU compatibility boundary.", + "Preserves representative pandas results and reports the validation command run or the runtime blocker encountered." + ] }, { "id": "cudf-csv-etl__generic", @@ -21,7 +25,11 @@ "evals/files/cudf-csv-etl/code/generate_data.py" ], "ground_truth": "A successful answer uses the provided cudf-csv-etl starter files, especially code/etl_pipeline.py, to move CSV read, filtering, computed columns, groupby aggregation, and parquet output to cuDF. It preserves filter predicates, computed-column formulas, grouping keys, aggregate columns, generated data paths, output paths, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-csv-etl starter workspace and edits code/etl_pipeline.py rather than inventing a new pipeline.", + "Moves CSV reading, filtering, computed columns, groupby aggregation, and parquet output to cuDF where supported.", + "Preserves filter predicates, computed-column formulas, grouping keys, aggregate outputs, generated data paths, and output paths, then reports validation or the runtime blocker." + ] }, { "id": "cudf-groupby-agg__generic", @@ -33,7 +41,11 @@ "evals/files/cudf-groupby-agg/code/groupby_analysis.py" ], "ground_truth": "A successful answer uses the provided cudf-groupby-agg starter files, especially code/groupby_analysis.py, to run the DataFrame loading and groupby work with cuDF. It preserves grouping keys, sum, mean, std, count, nunique, named aggregation, transform semantics or a documented compatibility boundary, output column names, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-groupby-agg starter workspace and edits code/groupby_analysis.py.", + "Runs DataFrame loading and groupby work with cuDF while preserving grouping keys, sum, mean, std, count, nunique, named aggregation, and output column names.", + "Handles transform semantics with cuDF or documents a compatibility boundary, and reports validation performed or the runtime blocker." + ] }, { "id": "cudf-multi-join__generic", @@ -45,7 +57,11 @@ "evals/files/cudf-multi-join/code/multi_join.py" ], "ground_truth": "A successful answer uses the provided cudf-multi-join starter files, especially code/multi_join.py, to migrate the orders, customers, and products joins plus downstream filtering and aggregation to cuDF. It preserves left and inner join types, join keys, suffix behavior, row-count expectations, post-join filters, output schema, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-multi-join starter workspace and edits code/multi_join.py.", + "Migrates the orders, customers, and products joins plus downstream filtering and aggregation to cuDF where supported.", + "Preserves join types, join keys, suffix behavior, row-count expectations, post-join filters, and output schema, then reports validation or the runtime blocker." + ] }, { "id": "cudf-null-handling__generic", @@ -57,7 +73,11 @@ "evals/files/cudf-null-handling/code/null_pipeline.py" ], "ground_truth": "A successful answer uses the provided cudf-null-handling starter files, especially code/null_pipeline.py, to move null detection, fill, drop, mask, and conditional fill logic to cuDF where supported. It preserves scalar and dictionary fill rules, subset and threshold drop rules, NA-aware boolean masks, interpolation or other compatibility boundaries, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-null-handling starter workspace and edits code/null_pipeline.py.", + "Moves null detection, fill, drop, mask, and conditional fill logic to cuDF where supported.", + "Preserves scalar and dictionary fill rules, subset and threshold drop rules, NA-aware masks, and any interpolation compatibility boundary, then reports validation or the runtime blocker." + ] }, { "id": "cudf-parquet-io__generic", @@ -69,7 +89,11 @@ "evals/files/cudf-parquet-io/code/parquet_pipeline.py" ], "ground_truth": "A successful answer uses the provided cudf-parquet-io starter files, especially code/parquet_pipeline.py, to migrate parquet reads, concatenation, filtering, column selection, dtype handling, and parquet writes to cuDF. It preserves multi-file input handling, partitioned output behavior, generated data paths, output paths, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-parquet-io starter workspace and edits code/parquet_pipeline.py.", + "Migrates parquet reads, concatenation, filtering, column selection, dtype handling, and parquet writes to cuDF where supported.", + "Preserves multi-file input handling, partitioned output behavior, generated data paths, and output paths, then reports validation or the runtime blocker." + ] }, { "id": "cudf-pivot-melt__generic", @@ -81,7 +105,11 @@ "evals/files/cudf-pivot-melt/code/reshape_analysis.py" ], "ground_truth": "A successful answer uses the provided cudf-pivot-melt starter files, especially code/reshape_analysis.py, to move supported reshape operations such as pivot, melt, stack/unstack, or crosstab-style logic to cuDF where practical. It preserves index labels, column labels, fill values, aggregation choices, output schema, compatibility boundaries, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-pivot-melt starter workspace and edits code/reshape_analysis.py.", + "Moves supported reshape operations such as pivot, melt, stack/unstack, or crosstab-style logic to cuDF where practical.", + "Preserves index labels, column labels, fill values, aggregation choices, output schema, and any compatibility boundary, then reports validation or the runtime blocker." + ] }, { "id": "cudf-string-ops__generic", @@ -93,7 +121,11 @@ "evals/files/cudf-string-ops/code/generate_data.py" ], "ground_truth": "A successful answer uses the provided cudf-string-ops starter files, especially code/clean_contacts.py, to migrate string cleaning to cuDF string accessors for lowercase, strip, contains, replace, and extract-style operations. It preserves regex patterns, extracted columns, null handling, string dtype behavior, representative cleaned values, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-string-ops starter workspace and edits code/clean_contacts.py.", + "Migrates string cleaning to cuDF string accessors for lowercase, strip, contains, replace, and extract-style operations where supported.", + "Preserves regex patterns, extracted columns, null handling, string dtype behavior, and representative cleaned values, then reports validation or the runtime blocker." + ] }, { "id": "cudf-timeseries-resample__generic", @@ -105,7 +137,11 @@ "evals/files/cudf-timeseries-resample/code/timeseries_analysis.py" ], "ground_truth": "A successful answer uses the provided cudf-timeseries-resample starter files, especially code/timeseries_analysis.py, to run datetime parsing, timestamp ordering, bucket creation, aggregation, and rolling computations with cuDF where supported. It preserves hourly and daily grouping semantics, missing buckets, rolling window sizes, output ordering, compatibility boundaries, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-timeseries-resample starter workspace and edits code/timeseries_analysis.py.", + "Runs datetime parsing, timestamp ordering, bucket creation, aggregation, and rolling computations with cuDF where supported.", + "Preserves hourly and daily grouping semantics, missing buckets, rolling window sizes, output ordering, and any compatibility boundary, then reports validation or the runtime blocker." + ] }, { "id": "cudf-window-functions__generic", @@ -117,7 +153,11 @@ "evals/files/cudf-window-functions/code/window_analysis.py" ], "ground_truth": "A successful answer uses the provided cudf-window-functions starter files, especially code/window_analysis.py, to migrate ranking, cumulative operations, rolling calculations, expanding calculations, and shift/lag work to cuDF where supported. It preserves group keys, ordering columns, rank methods, window sizes, edge and null behavior, output names, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-window-functions starter workspace and edits code/window_analysis.py.", + "Migrates ranking, cumulative operations, rolling calculations, expanding calculations, and shift/lag work to cuDF where supported.", + "Preserves group keys, ordering columns, rank methods, window sizes, edge and null behavior, and output names, then reports validation or the runtime blocker." + ] }, { "id": "source-cudf-null-fillna-semantics__generic", @@ -129,7 +169,11 @@ "evals/files/source-cudf-null-fillna-semantics/code/null_cleanup.py" ], "ground_truth": "A successful answer uses the provided source-cudf-null-fillna-semantics starter files, especially code/null_cleanup.py, to migrate the cleanup workflow to cuDF without changing missing-value meaning. It preserves nullable integer, string, category-like, mask/where, fillna, and groupby semantics without lossy sentinel conversions, includes or describes pandas-versus-cuDF parity validation, and reports validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided source-cudf-null-fillna-semantics starter workspace and edits code/null_cleanup.py.", + "Migrates the cleanup workflow to cuDF without changing missing-value meaning for nullable integers, strings, categories, masks, fillna, and groupby operations.", + "Includes or describes pandas-versus-cuDF parity validation and reports validation performed or the runtime blocker." + ] }, { "id": "cudf-native-stream-handoff-boundary__generic", @@ -142,7 +186,11 @@ "evals/files/cudf-native-stream-handoff-boundary/code/threaded_handoff.cu" ], "ground_truth": "A successful answer uses the provided cudf-native-stream-handoff-boundary starter files, especially code/threaded_handoff.cu, to fix cross-thread or cross-stream GPU handoff by tying CUDA event readiness to the object dependency. It orders consumer work after producer writes, orders destruction or free after last stream use, preserves asynchronous overlap where practical, and reports compile or smoke validation performed or the runtime blocker.", - "expected_behavior": [] + "expected_behavior": [ + "Uses the provided cudf-native-stream-handoff-boundary starter workspace and edits code/threaded_handoff.cu.", + "Fixes cross-thread or cross-stream GPU handoff by tying CUDA event readiness to the object dependency without adding unnecessary device-wide blocking.", + "Orders consumer work after producer writes and destruction/free after last stream use, then reports compile or smoke validation performed or the runtime blocker." + ] }, { "id": "negative-deep-learning-training__generic", @@ -153,6 +201,10 @@ "evals/files/negative-deep-learning-training/code/train.py" ], "ground_truth": "A successful answer treats the provided train.py context as a PyTorch/deep-learning training performance task rather than a cuDF migration. It keeps guidance focused on model training, data loading, batching, mixed precision, profiling, or other training-specific tactics, and only mentions cuDF as optional upstream tabular ETL when that is directly relevant.", - "expected_behavior": [] + "expected_behavior": [ + "Identifies the task as PyTorch or deep-learning training performance work rather than a cuDF DataFrame migration.", + "Keeps guidance focused on model training, data loading, batching, mixed precision, profiling, or other training-specific tactics.", + "Avoids invoking the cuDF skill except for an explicitly optional upstream tabular ETL note when directly relevant." + ] } ] diff --git a/skills/accelerated-computing-cudf/skill-card.md b/skills/accelerated-computing-cudf/skill-card.md index 5fd9de5e548a..66c8b180975c 100644 --- a/skills/accelerated-computing-cudf/skill-card.md +++ b/skills/accelerated-computing-cudf/skill-card.md @@ -9,7 +9,7 @@ NVIDIA
### License/Terms of Use:
CC-BY-4.0 AND Apache-2.0
## Use Case:
-Developers and engineers implementing GPU-accelerated DataFrame operations with NVIDIA cuDF, including pandas-to-GPU migration, ETL pipeline optimization, and multi-GPU workloads with dask-cuDF.
+Developers and engineers accelerating tabular data processing with GPU DataFrames, migrating pandas code to cuDF, optimizing ETL pipelines, and scaling DataFrame workloads across multiple GPUs.
### Deployment Geography for Use:
Global
@@ -28,9 +28,9 @@ Mitigation: Review and scan skill before deployment.
- [cuDF API Patterns, Gaps, and Semantic Differences](references/api-patterns.md)
- [cudf.pandas Accelerator Deep Dive](references/cudf-pandas-accelerator.md)
- [dask-cuDF Patterns](references/dask-cudf-patterns.md)
-- [cuDF Documentation](https://docs.rapids.ai/api/cudf/stable/)
-- [dask-cuDF API Reference](https://docs.rapids.ai/api/dask-cudf/stable/api/)
-- [cuDF GitHub Repository](https://github.com/NVIDIA/cudf)
+- [NVIDIA cuDF Documentation](https://docs.nvidia.com/cudf/)
+- [dask-cuDF Documentation](https://docs.nvidia.com/dask-cudf/)
+- [NVIDIA cuDF GitHub Repository](https://github.com/NVIDIA/cudf)
## Skill Output:
@@ -46,38 +46,39 @@ Mitigation: Review and scan skill before deployment.
## Evaluation Tasks:
-13 evaluation tasks (12 positive, 1 negative).
+13 evaluation tasks (12 positive, 1 negative), each run with 3 attempts in isolated sandbox pods.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Whether the skill avoids unsafe operations, secret leakage, and unauthorized access.
-- Correctness: Whether the final answer is correct against the reference answer.
-- Discoverability: Whether the expected skill is found and executed when needed.
-- Effectiveness: Whether the skill helps complete the user's goal and follows expected workflow.
-- Efficiency: Whether the skill avoids wasted tool or skill usage.
+- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Checks final-answer correctness against the reference answer.
+- Discoverability: Checks whether the expected skill was selected and the workflow executed.
+- Effectiveness: Checks whether the user’s goal was achieved and expected workflow behavior was followed.
+- Efficiency: Checks tool-call productivity and token usage efficiency.
Underlying evaluation signals used in this run:
-- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
+- `security`: Detects unsafe operations, secret leakage, and unauthorized access.
- `accuracy`: Verifies final-answer correctness against the reference answer.
-- `skill_execution`: Whether the expected skill was found and executed.
-- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
-- `goal_accuracy`: Whether the user's goal was achieved.
-- `behavior_check`: Whether the expected workflow behavior was followed.
+- `skill_execution`: Verifies whether the expected skill was selected and decoys were avoided.
+- `goal_accuracy`: Verifies whether the user’s goal was achieved.
+- `behavior_check`: Verifies whether the expected workflow behavior was followed.
+- `skill_efficiency`: Measures tool-call productivity.
+- `token_efficiency`: Measures actual uncached prompt plus completion token usage.
## Evaluation Results:
| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | |---|---:|---:| -| Overall | 74% → 88% (+14 points) | 68% → 83% (+15 points) | -| Security | 85% → 69% (-15 points) | 54% → 46% (-8 points) | -| Correctness | 100% → 100% (±0 points) | 100% → 98% (-2 points) | -| Discoverability | 47% → 89% (+42 points) | 45% → 85% (+40 points) | -| Effectiveness | 96% → 96% (±0 points) | 95% → 94% (-1 points) | -| Efficiency | 44% → 87% (+43 points) | 46% → 89% (+43 points) | +| Overall | Not available | 84.7% — baseline ran, but no comparable score was available; uplift unavailable | +| Security | Not available | 76.9% → 69.2% (-7.7 points) | +| Correctness | Not available | 100.0% → 100.0% (±0.0 points) | +| Discoverability | Not available | 81.3% — baseline ran, but no comparable score was available; uplift unavailable | +| Effectiveness | Not available | 94.4% → 90.9% (-3.5 points) | +| Efficiency | Not available | 82.3% — baseline ran, but no comparable score was available; uplift unavailable | ## Skill Version(s):
-333911cf41 (source: git SHA, committed 2026-08-26)
+4ad07b44f1 (source: git SHA, committed 2026-09-10)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/accelerated-computing-cudf/skill.oms.sig b/skills/accelerated-computing-cudf/skill.oms.sig index 4235e1b7b0d3..8ba376ec8db5 100644 --- a/skills/accelerated-computing-cudf/skill.oms.sig +++ b/skills/accelerated-computing-cudf/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYWNjZWxlcmF0ZWQtY29tcHV0aW5nLWN1ZGYiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYzBjNzY0MGY4N2FkYWNjNDQzODdiNDEzZjRiZTkxNDY3ZWM4ZGE1NzhkZTY5ZjU5NGUyNGRlYmY0ZmExMjQ1OCIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0aHViIgogICAgICBdLAogICAgICAibWV0aG9kIjogImZpbGVzIgogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiZGlnZXN0IjogImU1MzVmMWEzMTg0MTI2NTdkZmM2MTMyN2JlNTFhMjkwMzAxYmJmODlkMjM5NGQ4ODk0ZTY4MzJhNTgzNWUyY2MiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJkaWdlc3QiOiAiNWQzMTVlYjFkNjg1NTM2YWE0Mzc4ZjhmZWY3MDdhNDg5MzQxNjM1MTlkYzE5NjNmNWQ0NzNiNDQ1M2IzY2IxNiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiZGlnZXN0IjogImU5MGQyNDRlMDdkOGZmOWYzMzY1OTYyZDMzY2YyZjliZTJlNWNjZjFmOWJhYTUwZjVlYmIxOGY1YmFlMThiMTMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1hcHBseS11ZGYvY29kZS9nZW5lcmF0ZV9kYXRhLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjE5YTUyYTQ3NWNkOTZhNmFmZTFmOThlOWI5OTc1NGUxNTY2ZWJjY2IwODA0NzNkZDAyNDQwZDg3YjQyYmY4ZjAiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1hcHBseS11ZGYvY29kZS91ZGZfcGlwZWxpbmUucHkiLAogICAgICAgICJkaWdlc3QiOiAiYzE0ZGNkMzJhOGU2NGNhMzU5NDlmZjdmNjczYTlkZTdlZWE1NmRmNmRjOGJhMDgzMDNlM2FkODQ4MjAxZGIwZSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLWNzdi1ldGwvY29kZS9ldGxfcGlwZWxpbmUucHkiLAogICAgICAgICJkaWdlc3QiOiAiMzMyMTFlODFhZDhiY2ZlMDdlODJhYTA3NTFiMjE5Y2Q5OWQwMGE2YTcxZDZjNmJlMWFkZTY4NjdkZmJlMTY5NyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLWNzdi1ldGwvY29kZS9nZW5lcmF0ZV9kYXRhLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjk1ODcyNTdlN2ZiMDU2OTliYjllMzZiYWZmYTc0MzRiNGYzZmEyYmQ5NTQ1MmRkNmY0ZmNjNjRiNjIzYzkxMDMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1ncm91cGJ5LWFnZy9jb2RlL2dlbmVyYXRlX2RhdGEucHkiLAogICAgICAgICJkaWdlc3QiOiAiMWZkNjZhYmY3ZGNjOWUyNDFjYmY1Njg0YTJiYjczMmRkZTE2OGY0Mjg5ZWYzNmFmMmRkMjU3MmZhNDdiZDA1MiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLWdyb3VwYnktYWdnL2NvZGUvZ3JvdXBieV9hbmFseXNpcy5weSIsCiAgICAgICAgImRpZ2VzdCI6ICJlOWVmYTNhOGY4MjJjMjJiNmVhNzcwY2UyZDM3MzhjNDZkM2NmNDc2YzFlZmI2YTUxNjU4ODgyZGI1MTY4ZTM5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtbXVsdGktam9pbi9jb2RlL2dlbmVyYXRlX2RhdGEucHkiLAogICAgICAgICJkaWdlc3QiOiAiMzZmNGFkNjY3YmMwMDc5ODdhYWNlN2VmZTVmMTYxZDVkMDNkNDQ4ZDBkMjdlYzlmNzAxYjcyMGFjNDA5OWMyNiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLW11bHRpLWpvaW4vY29kZS9tdWx0aV9qb2luLnB5IiwKICAgICAgICAiZGlnZXN0IjogImVmMjg1MTcyOWJiMWE3NWE4ZTliM2NkZjY3MzNmZDE5MDZiNWI2MjE5NDc3YzVjMzMyYzZiMGQxM2UxNzY1NzQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1uYXRpdmUtc3RyZWFtLWhhbmRvZmYtYm91bmRhcnkvTk9USUNFLm1kIiwKICAgICAgICAiZGlnZXN0IjogImI5YTc3NTRhMTExODVjMTNlNDI0YjI0ZTMwZmFkYjY3ZjQwNzA5OWM0OTc4OGU0ZGQ1YjI1NTcwOTU0M2VjM2MiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1uYXRpdmUtc3RyZWFtLWhhbmRvZmYtYm91bmRhcnkvY29kZS9ydW5fc21va2Uuc2giLAogICAgICAgICJkaWdlc3QiOiAiM2YxYzEwZTNiMDcwMDViZWY4YmIxMDAzNTM3MTc3OTYzZjQ4Y2M1YmMwZWE4MmM5MTE5NWQ2NWU4OGM3NzhlZSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLW5hdGl2ZS1zdHJlYW0taGFuZG9mZi1ib3VuZGFyeS9jb2RlL3RocmVhZGVkX2hhbmRvZmYuY3UiLAogICAgICAgICJkaWdlc3QiOiAiNGQ0NGJkZmE0YmEyOTAxOTdkNGUxMmY5MzdlNjczMWQxOGYwN2Y0Y2U0ODkzNWNmNWFmNWQxZjAyNjE4NGE3NSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLW51bGwtaGFuZGxpbmcvY29kZS9nZW5lcmF0ZV9kYXRhLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjUzMWNlMTI3ZDE4ZWQ1MjI3NGJlMzY2NzE4ODBiODFhNWY1NWIwNWIxNTczZTE0NDkzM2JiNmE0OTczMzI2MWMiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1udWxsLWhhbmRsaW5nL2NvZGUvbnVsbF9waXBlbGluZS5weSIsCiAgICAgICAgImRpZ2VzdCI6ICJjN2FkYzE4NmZhYWFhZWE2MGM0NTEyOTA5MjYzMjk3NGI1YTRlZTlhMjc0MWIyODNhNDc4YTBkMWI3ODM1ODYwIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtcGFycXVldC1pby9jb2RlL2dlbmVyYXRlX2RhdGEucHkiLAogICAgICAgICJkaWdlc3QiOiAiZjk3OGI3NTEzNjQ1ZmIxM2U1NDAyZGIzN2FhNTk4YTk5OWQ4ZTAxMmIzZmVjYTNjOTA0MGM0ZmU2MjA2YzJjYyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLXBhcnF1ZXQtaW8vY29kZS9wYXJxdWV0X3BpcGVsaW5lLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjJiNzY1OGFhMjZkN2IyZGQ5ZmZhMDU4MGMzYjNmNjVlZTU5ZjJhODVlNGMwZWQxNzEyZGUzZmFhNDQ0MjMyYjIiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1waXZvdC1tZWx0L2NvZGUvZ2VuZXJhdGVfZGF0YS5weSIsCiAgICAgICAgImRpZ2VzdCI6ICJmODlkZDI3YWJlNjRiYzYzYjYxOTM3YTY3Yzc2MWNjOTk2OGQxM2JlOWY3OTI1ZjZmNzhiZWZiYTQ3Y2Q2NDYzIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtcGl2b3QtbWVsdC9jb2RlL3Jlc2hhcGVfYW5hbHlzaXMucHkiLAogICAgICAgICJkaWdlc3QiOiAiMzc3MjZiM2NmNDc2OGI0Y2Q2OGMwNTRlODI3NTdmNzU0MGZkNjZhYWY4ZDMyM2RlNmZlNjdmNjRmOTgwZDViOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLXN0cmluZy1vcHMvY29kZS9jbGVhbl9jb250YWN0cy5weSIsCiAgICAgICAgImRpZ2VzdCI6ICJiYzNkOTM3Y2U4N2YxMzJjMjE2NDRhY2RjNmFmNmI0MzI0NjFhNzJkYjIzYWJmMWY3ZGZjZTA2OTJjOTBlYzNhIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtc3RyaW5nLW9wcy9jb2RlL2dlbmVyYXRlX2RhdGEucHkiLAogICAgICAgICJkaWdlc3QiOiAiNzhmNDM5YzQzNDcwMmVhYzQ2YTM1ZmEyYmU4NTUyZTNiNjEyOTkyMGYyZmQ2YmFlYWFlOTA4YjJiZWRjNDQ2ZSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLXRpbWVzZXJpZXMtcmVzYW1wbGUvY29kZS9nZW5lcmF0ZV9kYXRhLnB5IiwKICAgICAgICAiZGlnZXN0IjogImNkNTg2MTI1ZjEyM2M0ZWJhY2U4OGNlNTFlOTgxM2RkOWU3ODY4NzhlNTU1NTRkYzkyMzAyZjA3NzQ0YjQ1YWYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi10aW1lc2VyaWVzLXJlc2FtcGxlL2NvZGUvdGltZXNlcmllc19hbmFseXNpcy5weSIsCiAgICAgICAgImRpZ2VzdCI6ICIxZGFjOTRkNGExNjcwYTI1NTFkN2YwZTAxNGEzMjU1OTljZTBmYzE4MTliOWIyMjBhZjJlNDEzMmQ5MTFlYWI2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtd2luZG93LWZ1bmN0aW9ucy9jb2RlL2dlbmVyYXRlX2RhdGEucHkiLAogICAgICAgICJkaWdlc3QiOiAiOTc2Yjc0MzQwNDA4ZmE3MjhkZGE3OTI0N2Y1NmU1OTkyMGQ2OTJmYzY1M2VlNzk5YTM0NjJjODU4M2NlMDZjNSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLXdpbmRvdy1mdW5jdGlvbnMvY29kZS93aW5kb3dfYW5hbHlzaXMucHkiLAogICAgICAgICJkaWdlc3QiOiAiMTE4NmQ0NDI0ZThiNWE5NTE0MzU2OGM2MjhkN2NkNTY5Yjc3MTk1ZTE3ZTdjZjUxOTBkZTcyNmJmMTAwYzdlMSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9uZWdhdGl2ZS1kZWVwLWxlYXJuaW5nLXRyYWluaW5nL2NvZGUvdHJhaW4ucHkiLAogICAgICAgICJkaWdlc3QiOiAiYzkzY2Q5ODU5MDc4NTlmNDc3MGM4ZmE0OWYxMTEyNmZhYjNlNjY2YmJkNDEyYmQyNGIxNTUyYzA0MTEwNmU4ZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9zb3VyY2UtY3VkZi1udWxsLWZpbGxuYS1zZW1hbnRpY3MvTk9USUNFLm1kIiwKICAgICAgICAiZGlnZXN0IjogImNjOGFlMjU2MGZkYTllZWFlYjdjNzFlMzQxZjllMDdkZmFmODczMWE3NTc3MTY2NzQ2OTkyODAzYWVmMzVlYWYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvc291cmNlLWN1ZGYtbnVsbC1maWxsbmEtc2VtYW50aWNzL2NvZGUvbnVsbF9jbGVhbnVwLnB5IiwKICAgICAgICAiZGlnZXN0IjogIjNmZWIyNWJjM2ViYWVkYWM4NTdlNmU3ZjA4NDJhNWI0M2ZkYzA4NDM2ZjI1NGNmZjViNGIxNDQ3NzdiZjQ4OWUiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9hcGktcGF0dGVybnMubWQiLAogICAgICAgICJkaWdlc3QiOiAiMDE0MDBjNzNiMjFiYjkzMWIzMjI5YmJiM2RhMmM5MDI0MWZmZDQ1NDRkZTZiYjA1MmI0OTEwM2Y0MjYxYjhkMyIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2N1ZGYtcGFuZGFzLWFjY2VsZXJhdG9yLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjBjMWVlNzk0OGIxZDgxOTFjZDI3ZGNhMTdiYjU1ZTdiOTU0NDJlM2FmNDc3YjVjMzQzY2MwMTgxNGMzNTJkM2YiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9kYXNrLWN1ZGYtcGF0dGVybnMubWQiLAogICAgICAgICJkaWdlc3QiOiAiOGE4ODlmMzE4ZDFiZjRmNWUwOTBjZDc5YTliMmZlMmI5Y2U0ZDI0NTljYjkxNDMzMjIwODNhYWFhNGZjYzcyYSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiZGlnZXN0IjogIjk2MzU4N2MwZjhjMDdhYzg3YzM5ZTJlYzM5ZWFjYmVkNDNmMjAxMTQwY2FhYTAwN2UzMGY2OWZhOTIxYjlhNmIiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDluIRhA+upyiYWNgYTrI3UjL7Tc4D7uY74AFEOZIZoJZ5XWoxsy8JpiJTPRGo3ZG4CMEoJh3yxExf9MlkarRMy3UrtEwfEO1fq7+f8nCOKKaJ0k1IZmyRzLpQc9NYwa70ESQ==","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYWNjZWxlcmF0ZWQtY29tcHV0aW5nLWN1ZGYiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiY2UxZjc3MWYyZjg2MmY1YzkzM2NmYWU2N2JmNzg4YjA5OWQ3YjgwMjFiZmEyMjRlNGNjYjNhNGY2MTQ0YmUyNyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfSwKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImFjMGFjZTNiZjkwYmVjNTgwMzA2YzExOTc4MmExNWIwZjcyZjIyN2I1MWViNDZmZTk1OTgxYzE4MDIyM2IzY2EiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImE3MTQxNjU4N2NlYmY5NGU4OGJlMDgxMjdiODU2ZjE0NWY0YzNiZGRjNGE1ZmJiZmU0MzhhMDE2Nzg1MGY5NDciLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiY2ExMDRkZjJjNmY2ZDIwNmU5ZTIyODIzNGE4YjA1NzljMGViZjljNTkwZjQxYTE1ZDE3OWUzMmIxYWE4NjdhOCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjE5YTUyYTQ3NWNkOTZhNmFmZTFmOThlOWI5OTc1NGUxNTY2ZWJjY2IwODA0NzNkZDAyNDQwZDg3YjQyYmY4ZjAiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLWFwcGx5LXVkZi9jb2RlL2dlbmVyYXRlX2RhdGEucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImMxNGRjZDMyYThlNjRjYTM1OTQ5ZmY3ZjY3M2E5ZGU3ZWVhNTZkZjZkYzhiYTA4MzAzZTNhZDg0ODIwMWRiMGUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLWFwcGx5LXVkZi9jb2RlL3VkZl9waXBlbGluZS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMzMyMTFlODFhZDhiY2ZlMDdlODJhYTA3NTFiMjE5Y2Q5OWQwMGE2YTcxZDZjNmJlMWFkZTY4NjdkZmJlMTY5NyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtY3N2LWV0bC9jb2RlL2V0bF9waXBlbGluZS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOTU4NzI1N2U3ZmIwNTY5OWJiOWUzNmJhZmZhNzQzNGI0ZjNmYTJiZDk1NDUyZGQ2ZjRmY2M2NGI2MjNjOTEwMyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtY3N2LWV0bC9jb2RlL2dlbmVyYXRlX2RhdGEucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjFmZDY2YWJmN2RjYzllMjQxY2JmNTY4NGEyYmI3MzJkZGUxNjhmNDI4OWVmMzZhZjJkZDI1NzJmYTQ3YmQwNTIiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLWdyb3VwYnktYWdnL2NvZGUvZ2VuZXJhdGVfZGF0YS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZTllZmEzYThmODIyYzIyYjZlYTc3MGNlMmQzNzM4YzQ2ZDNjZjQ3NmMxZWZiNmE1MTY1ODg4MmRiNTE2OGUzOSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtZ3JvdXBieS1hZ2cvY29kZS9ncm91cGJ5X2FuYWx5c2lzLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzNmY0YWQ2NjdiYzAwNzk4N2FhY2U3ZWZlNWYxNjFkNWQwM2Q0NDhkMGQyN2VjOWY3MDFiNzIwYWM0MDk5YzI2IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1tdWx0aS1qb2luL2NvZGUvZ2VuZXJhdGVfZGF0YS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZWYyODUxNzI5YmIxYTc1YThlOWIzY2RmNjczM2ZkMTkwNmI1YjYyMTk0NzdjNWMzMzJjNmIwZDEzZTE3NjU3NCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtbXVsdGktam9pbi9jb2RlL211bHRpX2pvaW4ucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImI5YTc3NTRhMTExODVjMTNlNDI0YjI0ZTMwZmFkYjY3ZjQwNzA5OWM0OTc4OGU0ZGQ1YjI1NTcwOTU0M2VjM2MiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLW5hdGl2ZS1zdHJlYW0taGFuZG9mZi1ib3VuZGFyeS9OT1RJQ0UubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjNmMWMxMGUzYjA3MDA1YmVmOGJiMTAwMzUzNzE3Nzk2M2Y0OGNjNWJjMGVhODJjOTExOTVkNjVlODhjNzc4ZWUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLW5hdGl2ZS1zdHJlYW0taGFuZG9mZi1ib3VuZGFyeS9jb2RlL3J1bl9zbW9rZS5zaCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNGQ0NGJkZmE0YmEyOTAxOTdkNGUxMmY5MzdlNjczMWQxOGYwN2Y0Y2U0ODkzNWNmNWFmNWQxZjAyNjE4NGE3NSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtbmF0aXZlLXN0cmVhbS1oYW5kb2ZmLWJvdW5kYXJ5L2NvZGUvdGhyZWFkZWRfaGFuZG9mZi5jdSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiNTMxY2UxMjdkMThlZDUyMjc0YmUzNjY3MTg4MGI4MWE1ZjU1YjA1YjE1NzNlMTQ0OTMzYmI2YTQ5NzMzMjYxYyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtbnVsbC1oYW5kbGluZy9jb2RlL2dlbmVyYXRlX2RhdGEucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImM3YWRjMTg2ZmFhYWFlYTYwYzQ1MTI5MDkyNjMyOTc0YjVhNGVlOWEyNzQxYjI4M2E0NzhhMGQxYjc4MzU4NjAiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLW51bGwtaGFuZGxpbmcvY29kZS9udWxsX3BpcGVsaW5lLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJmOTc4Yjc1MTM2NDVmYjEzZTU0MDJkYjM3YWE1OThhOTk5ZDhlMDEyYjNmZWNhM2M5MDQwYzRmZTYyMDZjMmNjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1wYXJxdWV0LWlvL2NvZGUvZ2VuZXJhdGVfZGF0YS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMmI3NjU4YWEyNmQ3YjJkZDlmZmEwNTgwYzNiM2Y2NWVlNTlmMmE4NWU0YzBlZDE3MTJkZTNmYWE0NDQyMzJiMiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtcGFycXVldC1pby9jb2RlL3BhcnF1ZXRfcGlwZWxpbmUucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImY4OWRkMjdhYmU2NGJjNjNiNjE5MzdhNjdjNzYxY2M5OTY4ZDEzYmU5Zjc5MjVmNmY3OGJlZmJhNDdjZDY0NjMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLXBpdm90LW1lbHQvY29kZS9nZW5lcmF0ZV9kYXRhLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzNzcyNmIzY2Y0NzY4YjRjZDY4YzA1NGU4Mjc1N2Y3NTQwZmQ2NmFhZjhkMzIzZGU2ZmU2N2Y2NGY5ODBkNWI4IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1waXZvdC1tZWx0L2NvZGUvcmVzaGFwZV9hbmFseXNpcy5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYmMzZDkzN2NlODdmMTMyYzIxNjQ0YWNkYzZhZjZiNDMyNDYxYTcyZGIyM2FiZjFmN2RmY2UwNjkyYzkwZWMzYSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtc3RyaW5nLW9wcy9jb2RlL2NsZWFuX2NvbnRhY3RzLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI3OGY0MzljNDM0NzAyZWFjNDZhMzVmYTJiZTg1NTJlM2I2MTI5OTIwZjJmZDZiYWVhYWU5MDhiMmJlZGM0NDZlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi1zdHJpbmctb3BzL2NvZGUvZ2VuZXJhdGVfZGF0YS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiY2Q1ODYxMjVmMTIzYzRlYmFjZTg4Y2U1MWU5ODEzZGQ5ZTc4Njg3OGU1NTU1NGRjOTIzMDJmMDc3NDRiNDVhZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtdGltZXNlcmllcy1yZXNhbXBsZS9jb2RlL2dlbmVyYXRlX2RhdGEucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjFkYWM5NGQ0YTE2NzBhMjU1MWQ3ZjBlMDE0YTMyNTU5OWNlMGZjMTgxOWI5YjIyMGFmMmU0MTMyZDkxMWVhYjYiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9jdWRmLXRpbWVzZXJpZXMtcmVzYW1wbGUvY29kZS90aW1lc2VyaWVzX2FuYWx5c2lzLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI5NzZiNzQzNDA0MDhmYTcyOGRkYTc5MjQ3ZjU2ZTU5OTIwZDY5MmZjNjUzZWU3OTlhMzQ2MmM4NTgzY2UwNmM1IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZmlsZXMvY3VkZi13aW5kb3ctZnVuY3Rpb25zL2NvZGUvZ2VuZXJhdGVfZGF0YS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMTE4NmQ0NDI0ZThiNWE5NTE0MzU2OGM2MjhkN2NkNTY5Yjc3MTk1ZTE3ZTdjZjUxOTBkZTcyNmJmMTAwYzdlMSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL2N1ZGYtd2luZG93LWZ1bmN0aW9ucy9jb2RlL3dpbmRvd19hbmFseXNpcy5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzkzY2Q5ODU5MDc4NTlmNDc3MGM4ZmE0OWYxMTEyNmZhYjNlNjY2YmJkNDEyYmQyNGIxNTUyYzA0MTEwNmU4ZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL25lZ2F0aXZlLWRlZXAtbGVhcm5pbmctdHJhaW5pbmcvY29kZS90cmFpbi5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiY2M4YWUyNTYwZmRhOWVlYWViN2M3MWUzNDFmOWUwN2RmYWY4NzMxYTc1NzcxNjY3NDY5OTI4MDNhZWYzNWVhZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogImV2YWxzL2ZpbGVzL3NvdXJjZS1jdWRmLW51bGwtZmlsbG5hLXNlbWFudGljcy9OT1RJQ0UubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjNmZWIyNWJjM2ViYWVkYWM4NTdlNmU3ZjA4NDJhNWI0M2ZkYzA4NDM2ZjI1NGNmZjViNGIxNDQ3NzdiZjQ4OWUiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9maWxlcy9zb3VyY2UtY3VkZi1udWxsLWZpbGxuYS1zZW1hbnRpY3MvY29kZS9udWxsX2NsZWFudXAucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjAxNDAwYzczYjIxYmI5MzFiMzIyOWJiYjNkYTJjOTAyNDFmZmQ0NTQ0ZGU2YmIwNTJiNDkxMDNmNDI2MWI4ZDMiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2FwaS1wYXR0ZXJucy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMGMxZWU3OTQ4YjFkODE5MWNkMjdkY2ExN2JiNTVlN2I5NTQ0MmUzYWY0NzdiNWMzNDNjYzAxODE0YzM1MmQzZiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvY3VkZi1wYW5kYXMtYWNjZWxlcmF0b3IubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhhODg5ZjMxOGQxYmY0ZjVlMDkwY2Q3OWE5YjJmZTJiOWNlNGQyNDU5Y2I5MTQzMzIyMDgzYWFhYTRmY2M3MmEiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2Rhc2stY3VkZi1wYXR0ZXJucy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiY2RlMmIyOTYyNGU1NWFhNDA5OTViN2MwYTY1ZGI3OTRlOWU4NmViYjJhMjM5MGI2MThjZmNmY2EyMDcwMDExZSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMD5E7FPsLml5fja2avUnGCxFB4WWu10WNhKUMSAg7vDhrDnB+hioGW1mIDOJ4rFHiQIwDLs5+Y3KeuOjBumkXp9VfG6dHWHCBUHuwozcudGNp7Kxxo/zhGNWniu0a39jpKCc","keyid":""}]}} \ No newline at end of file