Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,18 @@ public enum ColumnOrderName {
* The column order is defined by the IEEE 754 standard.
*/
IEEE_754_TOTAL_ORDER,
/**
* Chronological order for INT96 timestamps: values are compared by the Julian day (the last 4
* bytes, as a little-endian signed int32), then by the nanoseconds within the day (the first 8
* bytes, as a little-endian signed int64). Only supported for the INT96 physical type.
*/
INT96_TIMESTAMP_ORDER
}

private static final ColumnOrder UNDEFINED_COLUMN_ORDER = new ColumnOrder(ColumnOrderName.UNDEFINED);
private static final ColumnOrder TYPE_DEFINED_COLUMN_ORDER = new ColumnOrder(ColumnOrderName.TYPE_DEFINED_ORDER);
private static final ColumnOrder IEEE_754_TOTAL_ORDER = new ColumnOrder(ColumnOrderName.IEEE_754_TOTAL_ORDER);
private static final ColumnOrder INT96_TIMESTAMP_COLUMN_ORDER = new ColumnOrder(ColumnOrderName.INT96_TIMESTAMP_ORDER);

/**
* @return a {@link ColumnOrder} instance representing an undefined order
Expand All @@ -71,6 +78,14 @@ public static ColumnOrder ieee754TotalOrder() {
return IEEE_754_TOTAL_ORDER;
}

/**
* @return a {@link ColumnOrder} instance representing the chronological order of INT96 timestamps
* @see ColumnOrderName#INT96_TIMESTAMP_ORDER
*/
public static ColumnOrder int96TimestampOrder() {
return INT96_TIMESTAMP_COLUMN_ORDER;
}

private final ColumnOrderName columnOrderName;

private ColumnOrder(ColumnOrderName columnOrderName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import java.io.Serializable;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Comparator;
import org.apache.parquet.io.api.Binary;

Expand Down Expand Up @@ -354,4 +355,36 @@ public String toString() {
return "BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR";
}
};

/**
* Comparator for two timestamps encoded as INT96 (12-byte little-endian) binary.
* Layout: first 8 bytes = nanoseconds within the day, last 4 bytes = Julian day.
*
* Two-level comparison, matching the INT96 timestamp sort order:
* 1. Compare the last 4 bytes (Julian day) as a signed little-endian int32.
* 2. If equal, compare the first 8 bytes (nanos) as a signed little-endian int64.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The correctness of this depends on the two fields so I think we have to be more careful since the caller may pass in something unexpected.

I think that we need to validate that the nanosecond value is positive and less than the number of nanoseconds per day. If either of these is violated, then this sort order is no longer correct.

*/
static final PrimitiveComparator<Binary> BINARY_AS_INT96_TIMESTAMP_COMPARATOR = new BinaryComparator() {
@Override
int compareBinary(Binary b1, Binary b2) {
if (b1.length() != 12 || b2.length() != 12) {
throw new IllegalArgumentException(
"INT96 binary length must be 12, got " + b1.length() + " and " + b2.length());
}

ByteBuffer bb1 = b1.toByteBuffer().slice();
ByteBuffer bb2 = b2.toByteBuffer().slice();
bb1.order(ByteOrder.LITTLE_ENDIAN);
bb2.order(ByteOrder.LITTLE_ENDIAN);

int result = Integer.compare(bb1.getInt(8), bb2.getInt(8));
if (result != 0) return result;
return Long.compare(bb1.getLong(0), bb2.getLong(0));
}

@Override
public String toString() {
return "BINARY_AS_INT96_TIMESTAMP_COMPARATOR";
}
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,9 @@ public <T, E extends Exception> T convert(PrimitiveTypeNameConverter<T, E> conve

@Override
PrimitiveComparator<?> comparator(LogicalTypeAnnotation logicalType, ColumnOrder columnOrder) {
return PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR;
return columnOrder != null && columnOrder.getColumnOrderName() == ColumnOrderName.INT96_TIMESTAMP_ORDER
? PrimitiveComparator.BINARY_AS_INT96_TIMESTAMP_COMPARATOR
: PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR;
}
},
FIXED_LEN_BYTE_ARRAY("getBinary", Binary.class) {
Expand Down Expand Up @@ -578,9 +580,15 @@ public PrimitiveType(
this.decimalMeta = decimalMeta;

if (columnOrder == null) {
columnOrder = primitive == PrimitiveTypeName.INT96 || originalType == OriginalType.INTERVAL
? ColumnOrder.undefined()
: ColumnOrder.typeDefined();
if (primitive == PrimitiveTypeName.INT96) {
// A plain INT96 is the legacy timestamp encoding; default it to the chronological order.
// An annotated INT96 carries other semantics, so leave its order undefined.
columnOrder = originalType == null ? ColumnOrder.int96TimestampOrder() : ColumnOrder.undefined();

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that there are any original types that can be applied to INT96. We stopped adding them years ago. I think it would be safe to just return ColumnOrder.int96TimestampOrder() without the check. The comment would be "INT96 is only used for (deprecated) timestamps".

} else if (originalType == OriginalType.INTERVAL) {
columnOrder = ColumnOrder.undefined();
} else {
columnOrder = ColumnOrder.typeDefined();
}
} else if (columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER) {
Preconditions.checkArgument(
primitive == PrimitiveTypeName.FLOAT || primitive == PrimitiveTypeName.DOUBLE,
Expand Down Expand Up @@ -629,10 +637,17 @@ public PrimitiveType(
}

if (columnOrder == null) {
columnOrder = primitive == PrimitiveTypeName.INT96
|| logicalTypeAnnotation instanceof LogicalTypeAnnotation.IntervalLogicalTypeAnnotation
? ColumnOrder.undefined()
: ColumnOrder.typeDefined();
if (primitive == PrimitiveTypeName.INT96) {
// A plain INT96 is the legacy timestamp encoding; default it to the chronological order.
// An annotated INT96 carries other semantics, so leave its order undefined.
columnOrder = logicalTypeAnnotation == null
? ColumnOrder.int96TimestampOrder()
: ColumnOrder.undefined();
} else if (logicalTypeAnnotation instanceof LogicalTypeAnnotation.IntervalLogicalTypeAnnotation) {
columnOrder = ColumnOrder.undefined();
} else {
columnOrder = ColumnOrder.typeDefined();
}
} else if (columnOrder.getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER) {
Preconditions.checkArgument(
primitive == PrimitiveTypeName.FLOAT
Expand All @@ -651,9 +666,15 @@ public PrimitiveType(
private ColumnOrder requireValidColumnOrder(ColumnOrder columnOrder) {
if (primitive == PrimitiveTypeName.INT96) {
Preconditions.checkArgument(
columnOrder.getColumnOrderName() == ColumnOrderName.UNDEFINED,
columnOrder.getColumnOrderName() == ColumnOrderName.UNDEFINED
|| columnOrder.getColumnOrderName() == ColumnOrderName.INT96_TIMESTAMP_ORDER,
"The column order %s is not supported by INT96",
columnOrder);
} else {
Preconditions.checkArgument(
columnOrder.getColumnOrderName() != ColumnOrderName.INT96_TIMESTAMP_ORDER,
"The column order %s is only supported by INT96",
columnOrder);
}
if (getLogicalTypeAnnotation() != null) {
Preconditions.checkArgument(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,16 @@ public void testContractNonStringTypes() {
testTruncator(
Types.required(FIXED_LEN_BYTE_ARRAY).length(12).as(INTERVAL).named("test_fixed_interval"), false);
testTruncator(Types.required(BINARY).as(DECIMAL).precision(10).scale(2).named("test_binary_decimal"), false);
testTruncator(Types.required(INT96).named("test_int96"), false);

// INT96 has a fixed 12-byte width and a chronological comparator (so it is excluded from the
// variable-length checks above, like FLOAT16). Its truncator is a no-op: verify it returns the
// value unchanged regardless of the requested length.
BinaryTruncator int96Truncator = BinaryTruncator.getTruncator(

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is replacing the INT96 handling above, but this expansion deserves its own new test case because it is not running the standard testTrunctator logic. This should be in a new testInt96.

Types.required(INT96).named("test_int96"));
Binary int96Value = Binary.fromConstantByteArray(
new byte[] {0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4});
assertSame(int96Value, int96Truncator.truncateMin(int96Value, 4));
assertSame(int96Value, int96Truncator.truncateMax(int96Value, 4));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_FLOAT16_IEEE_754_TOTAL_ORDER_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_INT96_TIMESTAMP_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.BINARY_AS_SIGNED_INTEGER_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.BOOLEAN_COMPARATOR;
import static org.apache.parquet.schema.PrimitiveComparator.DOUBLE_COMPARATOR;
Expand All @@ -36,8 +37,12 @@

import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;
import org.apache.parquet.example.data.simple.NanoTime;
import org.apache.parquet.io.api.Binary;
import org.junit.Test;

Expand Down Expand Up @@ -354,6 +359,60 @@ public void testBinaryAsSignedIntegerComparatorWithEquals() {
}
}

private static Binary int96(int julianDay, long nanosOfDay) {
return new NanoTime(julianDay, nanosOfDay).toBinary();
}

private static Binary timestampToInt96(String timestamp) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see the value of complicating the tests with string parsing. I think you can test what you need to with Julian day and nanos. No need to make your tests depend on correct conversion from epoch to Julian.

LocalDateTime dt = LocalDateTime.parse(timestamp);
int julianDay = (int) (dt.toLocalDate().toEpochDay() + 2440588);
return new NanoTime(julianDay, dt.toLocalTime().toNanoOfDay()).toBinary();
}

@Test
public void testInt96TimestampComparator() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This also needs to cover cases like negative nanos and nanos >= 86_400_000_000_000. Use a separate test case if the intent is to fail.

Binary[] valuesInAscendingOrder = {
int96(Integer.MIN_VALUE, 0), // most negative julian day
int96(-1, 86_399_999_999_999L), // negative julian days sort before day 0
int96(0, 0), // start of the julian period
int96(0, 86_399_999_999_999L), // same day, later time of day
timestampToInt96("1968-05-23T00:00:00.000000123"), // pre-epoch but positive julian day
timestampToInt96("2020-01-01T12:00:00"),
timestampToInt96("2020-02-01T11:00:00"), // later day even though earlier time of day
timestampToInt96("2020-02-01T11:00:00.000000001"), // nanos tie-break
int96(Integer.MAX_VALUE, 86_399_999_999_999L)
};

// The same value in different Binary representations must compare identically; the offset
// variant guards against absolute reads not being relative to the value's start
List<Function<Binary, Binary>> representations = List.of(
b -> b,
b -> Binary.fromReusedByteArray(b.getBytes()),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's no functional difference between reused and constant here. It is just a way to inform Parquet that the value may be modified (and should be copied) or not.

I also don't think there is much value in these cases more generally. This is testing that Binary#toByteBuffer does the right thing and that is not the responsibility of this test to validate.

b -> Binary.fromConstantByteArray(b.getBytes()),
b -> {
byte[] bytes = b.getBytes();
byte[] padded = new byte[bytes.length + 20];
Arrays.fill(padded, (byte) 0xAA);
System.arraycopy(bytes, 0, padded, 10, bytes.length);
return Binary.fromReusedByteArray(padded, 10, bytes.length);
});

for (int i = 0; i < valuesInAscendingOrder.length; ++i) {
for (int j = 0; j < valuesInAscendingOrder.length; ++j) {
for (Function<Binary, Binary> fi : representations) {
for (Function<Binary, Binary> fj : representations) {
Binary bi = fi.apply(valuesInAscendingOrder[i]);
Binary bj = fj.apply(valuesInAscendingOrder[j]);
assertEquals(
"comparing value " + i + " to value " + j,
Integer.signum(Integer.compare(i, j)),
Integer.signum(BINARY_AS_INT96_TIMESTAMP_COMPARATOR.compare(bi, bj)));
}
}
}
}
}

@Test
public void testFloat16Comparator() {
Binary[] valuesInAscendingOrder = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import org.apache.parquet.format.GeometryType;
import org.apache.parquet.format.GeospatialStatistics;
import org.apache.parquet.format.IEEE754TotalOrder;
import org.apache.parquet.format.Int96TimestampOrder;
import org.apache.parquet.format.IntType;
import org.apache.parquet.format.KeyValue;
import org.apache.parquet.format.LogicalType;
Expand Down Expand Up @@ -146,6 +147,7 @@ public class ParquetMetadataConverter {

private static final TypeDefinedOrder TYPE_DEFINED_ORDER = new TypeDefinedOrder();
private static final IEEE754TotalOrder IEEE_754_TOTAL_ORDER = new IEEE754TotalOrder();
private static final Int96TimestampOrder INT96_TIMESTAMP_ORDER = new Int96TimestampOrder();
public static final MetadataFilter NO_FILTER = new NoFilter();
public static final MetadataFilter SKIP_ROW_GROUPS = new SkipMetadataFilter();
public static final long MAX_STATS_SIZE = 4096; // limit stats to 4k
Expand Down Expand Up @@ -290,6 +292,9 @@ private List<ColumnOrder> getColumnOrders(MessageType schema) {
case IEEE_754_TOTAL_ORDER:
columnOrder.setIEEE_754_TOTAL_ORDER(IEEE_754_TOTAL_ORDER);
break;
case INT96_TIMESTAMP_ORDER:
columnOrder.setINT96_TIMESTAMP_ORDER(INT96_TIMESTAMP_ORDER);
break;
case UNDEFINED:
// Use TypeDefinedOrder if some types (e.g. INT96) have undefined column orders.
columnOrder.setTYPE_ORDER(TYPE_DEFINED_ORDER);
Expand Down Expand Up @@ -911,8 +916,10 @@ private static byte[] tuncateMax(BinaryTruncator truncator, int truncateLength,
}

private static boolean isMinMaxStatsSupported(PrimitiveType type) {
return type.columnOrder().getColumnOrderName() == ColumnOrderName.TYPE_DEFINED_ORDER
|| type.columnOrder().getColumnOrderName() == ColumnOrderName.IEEE_754_TOTAL_ORDER;
ColumnOrderName name = type.columnOrder().getColumnOrderName();
return name == ColumnOrderName.TYPE_DEFINED_ORDER
|| name == ColumnOrderName.IEEE_754_TOTAL_ORDER
|| name == ColumnOrderName.INT96_TIMESTAMP_ORDER;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is getting too complicated. Either use a switch or use name != UNDEFINED.

}

/**
Expand Down Expand Up @@ -2057,7 +2064,17 @@ private void buildChildren(
|| schemaElement.converted_type == ConvertedType.INTERVAL)) {
columnOrder = org.apache.parquet.schema.ColumnOrder.undefined();
}
// INT96_TIMESTAMP_ORDER is only valid for INT96 columns, ignore it anywhere else.
if (columnOrder.getColumnOrderName() == ColumnOrderName.INT96_TIMESTAMP_ORDER
&& schemaElement.type != Type.INT96) {
columnOrder = org.apache.parquet.schema.ColumnOrder.undefined();
}
primitiveBuilder.columnOrder(columnOrder);
} else if (schemaElement.type == Type.INT96) {
// A footer without column orders predates INT96_TIMESTAMP_ORDER, so an INT96 column here
// must not inherit the (chronological) construction-time default: its stats, if any, were
// written under the legacy order and must be ignored.
primitiveBuilder.columnOrder(org.apache.parquet.schema.ColumnOrder.undefined());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There isn't a test in TestParquetMetadataConverter that validates Parquet metadata for an INT96 without a column order. There is a test that a new schema will produce a column order, but not one to verify that a schema without one will be read correctly.

}
childBuilder = primitiveBuilder;
} else {
Expand Down Expand Up @@ -2110,6 +2127,9 @@ private static org.apache.parquet.schema.ColumnOrder fromParquetColumnOrder(Colu
if (columnOrder.isSetIEEE_754_TOTAL_ORDER()) {
return org.apache.parquet.schema.ColumnOrder.ieee754TotalOrder();
}
if (columnOrder.isSetINT96_TIMESTAMP_ORDER()) {
return org.apache.parquet.schema.ColumnOrder.int96TimestampOrder();
}
// The column order is not yet supported by this API
return org.apache.parquet.schema.ColumnOrder.undefined();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1171,17 +1171,14 @@ public void testMissingValuesFromStats() {

@Test
public void testSkippedV2Stats() {
// INTERVAL has an undefined column order, so its stats are skipped.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove this. It is unrelated and doesn't need to be part of this PR.

testSkippedV2Stats(
Types.optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
.length(12)
.as(OriginalType.INTERVAL)
.named(""),
new BigInteger("12345678"),
new BigInteger("12345679"));
testSkippedV2Stats(
Types.optional(PrimitiveTypeName.INT96).named(""),
new BigInteger("-75687987"),
new BigInteger("45367657"));

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are other tests that also need to be updated. For example, testV2StatsEqualMinMax tests cases where the min and max are equal and stats are preserved. This also uses BigInteger to pass the values, which is suspicious.

I think this should update all of the INT96 cases to use NanoTime. Looking at the code path, I don't think that NanoTime is plumbed to work since I don't see a Statistics implementation for it. Maybe this should actually pass a Binary produced by NanoTime instead.

}

private void testSkippedV2Stats(PrimitiveType type, Object min, Object max) {
Expand Down Expand Up @@ -1383,8 +1380,7 @@ public void testColumnOrders() throws IOException {
+ " required binary key (UTF8);" // Key to be hacked to have unknown column order -> undefined
+ " optional group list_col (LIST) {"
+ " repeated group list {"
+ " optional int96 array_element;" // INT96 element with type defined column order ->
// undefined
+ " optional int96 array_element;" // plain INT96 element -> INT96_TIMESTAMP_ORDER
+ " }"
+ " }"
+ " }"
Expand All @@ -1398,9 +1394,10 @@ public void testColumnOrders() throws IOException {

List<org.apache.parquet.format.ColumnOrder> columnOrders = formatMetadata.getColumn_orders();
assertEquals(3, columnOrders.size());
for (org.apache.parquet.format.ColumnOrder columnOrder : columnOrders) {
assertTrue(columnOrder.isSetTYPE_ORDER());
}
// binary_col and key get TYPE_ORDER, the INT96 array_element gets INT96_TIMESTAMP_ORDER.
assertTrue(columnOrders.get(0).isSetTYPE_ORDER());
assertTrue(columnOrders.get(1).isSetTYPE_ORDER());
assertTrue(columnOrders.get(2).isSetINT96_TIMESTAMP_ORDER());

// Simulate that thrift got a union type that is not in the generated code
// (when the file contains a not-yet-supported column order)
Expand All @@ -1413,7 +1410,7 @@ public void testColumnOrders() throws IOException {
assertEquals(
ColumnOrder.typeDefined(), columns.get(0).getPrimitiveType().columnOrder());
assertEquals(ColumnOrder.undefined(), columns.get(1).getPrimitiveType().columnOrder());
assertEquals(ColumnOrder.undefined(), columns.get(2).getPrimitiveType().columnOrder());
assertEquals(ColumnOrder.int96TimestampOrder(), columns.get(2).getPrimitiveType().columnOrder());
}

@Test
Expand Down Expand Up @@ -1488,7 +1485,11 @@ public void testColumnIndexConversion() {
assertNull(
"Should ignore unsupported types",
ParquetMetadataConverter.toParquetColumnIndex(
Types.required(PrimitiveTypeName.INT96).named("test_int96"), columnIndex));
Types.required(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY)
.length(12)
.as(OriginalType.INTERVAL)
.named("test_interval"),
columnIndex));
assertNull(
"Should ignore unsupported types",
ParquetMetadataConverter.fromParquetColumnIndex(
Expand Down
Loading
Loading