Skip to content

Implement bound truncation in parquet writer #29164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ballard26 wants to merge 10 commits into
base: dev
Choose a base branch
from
+1,005 −110
Draft

Implement bound truncation in parquet writer #29164

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
6791d83
bytes: allow for pointers to all byte-sized types in io_placeholder
ballard26 Jan 7, 2026
81b6f4d
bytes: expose reverse fragment iterator in iobuf
ballard26 Jan 7, 2026
3abadda
bytes: implement reverse byte iterator for iobuf
ballard26 Jan 7, 2026
6262bca
bytes: fix case where the last rhs fragment is skipped during comparison
ballard26 Jan 7, 2026
b004a33
strings: add additional utils for utf-encoded byte sequences
ballard26 Jan 7, 2026
af2da8f
serde/parquet: use zero-copy for bound encoding
ballard26 Jan 7, 2026
6cccc78
test_utils: make bound checks more permissive in parquet_verifier
ballard26 Jan 7, 2026
96e9f3e
serde/parquet: add min/max value truncators for byte array types
ballard26 Jan 7, 2026
9ce1d1a
serde/parquet: integrate bound truncation into parquet writer
ballard26 Jan 7, 2026
c3837e1
serde/parquet/tests: increase size of generated byte array fields
ballard26 Jan 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 60 additions & 23 deletions src/v/bytes/details/io_byte_iterator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,18 @@
#include "bytes/details/io_fragment.h"
Copy link
Member

@travisdowns travisdowns Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

iobuf_fuzz should probably be augemnted with reverse iterator cases


#include <iterator>
#include <type_traits>

// See io_iterator_consumer for iterator validity notes.
namespace details {
class io_byte_iterator {

template<bool Forward>
class io_byte_iterator_base {
public:
using io_const_iterator = io_fragment_list::const_iterator;
using io_const_iterator = std::conditional_t<
Forward,
io_fragment_list::const_iterator,
io_fragment_list::const_reverse_iterator>;

// iterator_traits
using difference_type = void;
Expand All @@ -28,24 +34,35 @@ class io_byte_iterator {
using reference = const char&;
using iterator_category = std::forward_iterator_tag;

io_byte_iterator(
io_byte_iterator_base(
const io_const_iterator& begin, const io_const_iterator& end) noexcept
: _frag(begin)
, _frag_end(end) {
if (_frag != _frag_end) {
_frag_index = _frag->get();
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
_frag_index_end = _frag->get() + _frag->size();
// handle an empty fragment
if (_frag_index == _frag_index_end) {
next_fragment();
if constexpr (Forward) {
_frag_index = _frag->get();
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
_frag_index_end = _frag->get() + _frag->size();
// handle an empty fragment
if (_frag_index == _frag_index_end) {
next_fragment();
}
} else {
auto frag_size = _frag->size();
if (frag_size == 0) {
next_fragment();
return;
}
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
_frag_index = _frag->get() + (_frag->size() - 1);
_frag_index_end = _frag->get();
}
} else {
_frag_index = nullptr;
_frag_index_end = nullptr;
}
}
io_byte_iterator(
io_byte_iterator_base(
const io_const_iterator& begin,
const io_const_iterator& end,
const char* frag_index,
Expand All @@ -59,20 +76,27 @@ class io_byte_iterator {
reference operator*() const noexcept { return *_frag_index; }
pointer operator->() const noexcept { return _frag_index; }
/// true if pointing to the byte-value (not necessarily the same address)
bool operator==(const io_byte_iterator& o) const noexcept {
bool operator==(const io_byte_iterator_base& o) const noexcept {
return _frag_index == o._frag_index;
}
bool operator!=(const io_byte_iterator& o) const noexcept {
bool operator!=(const io_byte_iterator_base& o) const noexcept {
return !(*this == o);
}
io_byte_iterator& operator++() {
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
if (++_frag_index == _frag_index_end) {
next_fragment();
io_byte_iterator_base& operator++() {
if constexpr (Forward) {
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
if (++_frag_index == _frag_index_end) {
next_fragment();
}
} else {
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
if (_frag_index-- == _frag_index_end) {
Copy link
Member

@travisdowns travisdowns Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This will form a "one before the start" pointer (when the condition is true), which is not allowed (yes, it's quite annoying).

next_fragment();
}
}
return *this;
}
io_byte_iterator operator++(int) {
io_byte_iterator_base operator++(int) {
auto tmp = *this;
++*this;
return tmp;
Expand All @@ -83,12 +107,22 @@ class io_byte_iterator {
while (true) {
++_frag;
if (_frag != _frag_end) {
_frag_index = _frag->get();
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
_frag_index_end = _frag->get() + _frag->size();
// handle an empty fragment
if (_frag_index == _frag_index_end) {
continue;
if constexpr (Forward) {
Copy link
Member

@travisdowns travisdowns Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This block of logic should be DRY with the identical block of logic in the ctor, probably called "maybe_next_fragment", called in the ctor and after every increment/decrement type operation.

_frag_index = _frag->get();
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
_frag_index_end = _frag->get() + _frag->size();
// handle an empty fragment
if (_frag_index == _frag_index_end) {
continue;
}
} else {
auto frag_size = _frag->size();
if (frag_size == 0) {
continue;
}
// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
_frag_index = _frag->get() + (frag_size - 1);
_frag_index_end = _frag->get();
}
return;
}
Expand All @@ -104,4 +138,7 @@ class io_byte_iterator {
const char* _frag_index_end = nullptr;
};

using io_byte_iterator = io_byte_iterator_base<true>;
using reverse_io_byte_iterator = io_byte_iterator_base<false>;

} // namespace details
4 changes: 4 additions & 0 deletions src/v/bytes/details/io_fragment.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ class io_fragment_list {
using iterator = iter<false, true>;
using reverse_iterator = iter<false, false>;
using const_iterator = iter<true, true>;
using const_reverse_iterator = iter<true, false>;

bool empty() const {
check_consistency();
Expand Down Expand Up @@ -338,11 +339,14 @@ class io_fragment_list {
reverse_iterator rend() { return {this, nullptr}; }
const_iterator cbegin() const { return {this, _head}; }
const_iterator cend() const { return {this, nullptr}; }
const_reverse_iterator crbegin() const { return {this, _tail}; }
const_reverse_iterator crend() const { return {this, nullptr}; }

private:
friend class iter<true, true>;
friend class iter<false, true>;
friend class iter<false, false>;
friend class iter<true, false>;
inline void update_generation() {
#ifndef NDEBUG
++_generation;
Expand Down
5 changes: 4 additions & 1 deletion src/v/bytes/details/io_placeholder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ class io_placeholder {
, _byte_index(initial_index)
, _remaining_size(max_size_to_write) {}

[[gnu::always_inline]] void write(const char* src, size_t len) {
template<typename T>
[[gnu::always_inline]] void write(const T* src, size_t len)
requires(sizeof(T) == 1)
Copy link
Member

@travisdowns travisdowns Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This function needs a doc now that it's not entirely obvious what it is doing.

It will do an element-wise copy from the src array to this placeholder, if T can be assigned to a char, right?

{
details::check_out_of_range(len, _remaining_size);
std::copy_n(src, len, mutable_index());
_remaining_size -= len;
Expand Down
10 changes: 10 additions & 0 deletions src/v/bytes/iobuf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,10 @@ class iobuf {
using iterator = typename container::iterator;
using reverse_iterator = typename container::reverse_iterator;
using const_iterator = typename container::const_iterator;
using const_reverse_iterator = typename container::const_reverse_iterator;
using iterator_consumer = details::io_iterator_consumer;
using byte_iterator = details::io_byte_iterator;
using reverse_byte_iterator = details::reverse_io_byte_iterator;
using placeholder = details::io_placeholder;

static iobuf from(std::string_view view) {
Expand Down Expand Up @@ -268,6 +270,8 @@ class iobuf {
const_iterator end() const;
const_iterator cbegin() const;
const_iterator cend() const;
const_reverse_iterator crbegin() const;
const_reverse_iterator crend() const;

std::string hexdump(size_t) const;

Expand Down Expand Up @@ -307,6 +311,12 @@ inline iobuf::const_iterator iobuf::begin() const { return _frags.cbegin(); }
inline iobuf::const_iterator iobuf::end() const { return _frags.cend(); }
inline iobuf::const_iterator iobuf::cbegin() const { return _frags.cbegin(); }
inline iobuf::const_iterator iobuf::cend() const { return _frags.cend(); }
inline iobuf::const_reverse_iterator iobuf::crbegin() const {
return _frags.crbegin();
}
inline iobuf::const_reverse_iterator iobuf::crend() const {
return _frags.crend();
}

inline bool iobuf::operator!=(const iobuf& o) const { return !(*this == o); }
inline bool iobuf::operator!=(std::string_view o) const {
Expand Down
46 changes: 46 additions & 0 deletions src/v/bytes/tests/iobuf_tests.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,52 @@ SEASTAR_THREAD_TEST_CASE(traver_all_bytes_one_at_a_time) {
}
BOOST_CHECK_EQUAL(str, expected.data());
}
SEASTAR_THREAD_TEST_CASE(iterate_bytes_backward_no_frags) {
auto b = iobuf();
auto begin = iobuf::reverse_byte_iterator(b.crbegin(), b.crend());
auto end = iobuf::reverse_byte_iterator(b.crend(), b.crend());
size_t bytes_read = 0;
for (; begin != end; begin++) {
bytes_read++;
}
BOOST_CHECK_EQUAL(bytes_read, 0);
}
SEASTAR_THREAD_TEST_CASE(iterate_bytes_backward_empty_frags) {
auto b = iobuf();
b.append(std::make_unique<iobuf::fragment>(0));
b.append(std::make_unique<iobuf::fragment>(1));
auto begin = iobuf::reverse_byte_iterator(b.crbegin(), b.crend());
auto end = iobuf::reverse_byte_iterator(b.crend(), b.crend());
size_t bytes_read = 0;
for (; begin != end; begin++) {
bytes_read++;
}
BOOST_CHECK_EQUAL(bytes_read, 0);
}
SEASTAR_THREAD_TEST_CASE(iterate_bytes_backward_mult_frags) {
auto b = iobuf();
std::string str1 = "hello";
std::string str2 = "world";

b.append(
std::make_unique<iobuf::fragment>(
ss::temporary_buffer<char>::copy_of(str1)));
b.append(std::make_unique<iobuf::fragment>(1));
b.append(
std::make_unique<iobuf::fragment>(
ss::temporary_buffer<char>::copy_of(str2)));

auto begin = iobuf::reverse_byte_iterator(b.crbegin(), b.crend());
auto end = iobuf::reverse_byte_iterator(b.crend(), b.crend());

iobuf rev;
for (; begin != end; begin++) {
rev.append(&*begin, 1);
}
BOOST_CHECK_EQUAL(
rev.linearize_to_string(),
std::ranges::to<std::string>(std::views::reverse(str1 + str2)));
}
SEASTAR_THREAD_TEST_CASE(not_equal_by_size) {
auto a = iobuf();
auto b = iobuf();
Expand Down