Skip to content

Implement bound truncation in parquet writer #29164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ballard26 wants to merge 10 commits into
base: dev
Choose a base branch
from
+1,005 −110
Draft

Implement bound truncation in parquet writer #29164

Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
6791d83
bytes: allow for pointers to all byte-sized types in io_placeholder
ballard26 Jan 7, 2026
81b6f4d
bytes: expose reverse fragment iterator in iobuf
ballard26 Jan 7, 2026
3abadda
bytes: implement reverse byte iterator for iobuf
ballard26 Jan 7, 2026
6262bca
bytes: fix case where the last rhs fragment is skipped during comparison
ballard26 Jan 7, 2026
b004a33
strings: add additional utils for utf-encoded byte sequences
ballard26 Jan 7, 2026
af2da8f
serde/parquet: use zero-copy for bound encoding
ballard26 Jan 7, 2026
6cccc78
test_utils: make bound checks more permissive in parquet_verifier
ballard26 Jan 7, 2026
96e9f3e
serde/parquet: add min/max value truncators for byte array types
ballard26 Jan 7, 2026
9ce1d1a
serde/parquet: integrate bound truncation into parquet writer
ballard26 Jan 7, 2026
c3837e1
serde/parquet/tests: increase size of generated byte array fields
ballard26 Jan 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 69 additions & 18 deletions src/v/test_utils/go/parquet_verifier/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,18 +163,47 @@ func main() {
i, j, col.NumValues(), col.NullCount(),
Copy link
Member

@travisdowns travisdowns Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is allowed in the parquet format? I.e., it is not required that the min/max actually exist in the page, just that they are bounds?

)
}
if !parquet.Equal(cmin, colMin) {
if len(cmin.Bytes()) > 64 || len(cmax.Bytes()) > 64 {
log.Fatalf(
"❌ : invalid min value for page in row group %d, column %d: %v != %v\n",
i, j, cmin, colMin,
"❌ : invalid length for cmin %v(len %d) or cmax %v(len %d) needs to be <= 64 bytes\n",
cmin, len(cmin.Bytes()), cmax, len(cmax.Bytes()),
)
}
if !parquet.Equal(cmax, colMax) {
log.Fatalf(
"❌ : invalid max value for page in row group %d, column %d: %v != %v\n",
i, j, cmax, colMax,
)

switch cmin.Kind() {
case parquet.ByteArray, parquet.FixedLenByteArray:
if parquet.ByteArrayType.Compare(cmin, colMin) > 0 {
log.Fatalf(
"❌ : invalid min value for page in row group %d, column %d: %v > %v\n",
i, j, cmin, colMin,
)
}
default:
if !parquet.Equal(cmin, colMin) {
log.Fatalf(
"❌ : invalid min value for page in row group %d, column %d: %v != %v\n",
i, j, cmin, colMin,
)
}
}

switch cmax.Kind() {
case parquet.ByteArray, parquet.FixedLenByteArray:
if parquet.ByteArrayType.Compare(cmax, colMax) < 0 {
log.Fatalf(
"❌ : invalid max value for page in row group %d, column %d: %v < %v\n",
i, j, cmax, colMax,
)
}
default:
if !parquet.Equal(cmax, colMax) {
log.Fatalf(
"❌ : invalid max value for page in row group %d, column %d: %v != %v\n",
i, j, cmax, colMax,
)
}
}

_ = pages.Close()
}
}
Expand Down Expand Up @@ -233,18 +262,40 @@ func main() {
)
}

if !reflect.DeepEqual(omin, rmin) {
log.Fatalf(
"❌ row group %d, column %d min value mismatch: %v != %v\n",
i, j, omin, rmin,
)
// Note that parquet-go currently does not truncate bounds in page stats.
// Hence the bounds will not be equal in the rewritten file.
switch omin.Kind() {
case parquet.ByteArray, parquet.FixedLenByteArray:
if parquet.ByteArrayType.Compare(omin, rmin) > 0 {
log.Fatalf(
"❌ row group %d, column %d min value mismatch: %v > %v\n",
i, j, omin, rmin,
)
}
default:
if !parquet.Equal(omin, rmin) {
log.Fatalf(
"❌ row group %d, column %d min value mismatch: %v != %v\n",
i, j, omin, rmin,
)
}
}

if !reflect.DeepEqual(omax, rmax) {
log.Fatalf(
"❌ row group %d, column %d max value mismatch: %v != %v\n",
i, j, omax, rmin,
)
switch omax.Kind() {
case parquet.ByteArray, parquet.FixedLenByteArray:
if parquet.ByteArrayType.Compare(omax, rmax) < 0 {
log.Fatalf(
"❌ row group %d, column %d max value mismatch: %v < %v\n",
i, j, omax, rmax,
)
}
default:
if !parquet.Equal(omax, rmax) {
log.Fatalf(
"❌ row group %d, column %d max value mismatch: %v != %v\n",
i, j, omax, rmin,
)
}
}
}
}
Expand Down