From 86fe3b5b4ad241e0d538ef23155642c62ac45204 Mon Sep 17 00:00:00 2001 From: Grant Nicholas Date: Wed, 3 Dec 2025 11:59:54 -0600 Subject: [PATCH 1/2] Support avro timestamp-nanos logical type Only support in native block handler, not hive block handler --- .../avro/HiveAvroTypeBlockHandler.java | 5 +- ...ativeLogicalTypesAvroTypeBlockHandler.java | 35 ++++ .../NativeLogicalTypesAvroTypeManager.java | 29 ++- .../formats/avro/model/AvroLogicalType.java | 11 ++ ...ataReaderWithAvroNativeTypeManagement.java | 170 ++++++++++++++++-- .../product/hive/TestAvroSchemaLiteral.java | 83 +++++++++ 6 files changed, 318 insertions(+), 15 deletions(-) diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/HiveAvroTypeBlockHandler.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/HiveAvroTypeBlockHandler.java index 64d0bbb9c5d2..e3f5db9c1fde 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/HiveAvroTypeBlockHandler.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/HiveAvroTypeBlockHandler.java @@ -30,6 +30,7 @@ import io.trino.hive.formats.avro.model.AvroLogicalType.TimeMillisLogicalType; import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampMicrosLogicalType; import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampMillisLogicalType; +import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampNanosLogicalType; import io.trino.hive.formats.avro.model.AvroReadAction; import io.trino.hive.formats.avro.model.AvroReadAction.LongIoFunction; import io.trino.hive.formats.avro.model.BooleanRead; @@ -149,7 +150,7 @@ else if (schema.getType() == Schema.Type.UNION) { case BytesDecimalLogicalType bytesDecimalLogicalType -> getAvroLogicalTypeSpiType(bytesDecimalLogicalType); case TimestampMillisLogicalType _ -> hiveSessionTimestamp; // Other logical types ignored by hive/don't map to hive types - case FixedDecimalLogicalType _, TimeMicrosLogicalType _, TimeMillisLogicalType _, TimestampMicrosLogicalType _, StringUUIDLogicalType _ -> baseTypeFor(schema, this); + case FixedDecimalLogicalType _, TimeMicrosLogicalType _, TimeMillisLogicalType _, TimestampMicrosLogicalType _, TimestampNanosLogicalType _, StringUUIDLogicalType _ -> baseTypeFor(schema, this); }; }; } @@ -191,7 +192,7 @@ public BlockBuildingDecoder blockBuildingDecoderFor(AvroReadAction readAction) // Hive only supports Bytes Decimal Type case FixedDecimalLogicalType _ -> baseBlockBuildingDecoderWithUnionCoerceAndErrorDelays(readAction); // Other logical types ignored by hive/don't map to hive types - case TimeMicrosLogicalType _, TimeMillisLogicalType _, TimestampMicrosLogicalType _, StringUUIDLogicalType _ -> baseBlockBuildingDecoderWithUnionCoerceAndErrorDelays(readAction); + case TimeMicrosLogicalType _, TimeMillisLogicalType _, TimestampMicrosLogicalType _, TimestampNanosLogicalType _, StringUUIDLogicalType _ -> baseBlockBuildingDecoderWithUnionCoerceAndErrorDelays(readAction); }; }; } diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeBlockHandler.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeBlockHandler.java index 0dabc8036e9f..7dc8bc464e0e 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeBlockHandler.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeBlockHandler.java @@ -23,6 +23,7 @@ import io.trino.hive.formats.avro.model.AvroLogicalType.TimeMillisLogicalType; import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampMicrosLogicalType; import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampMillisLogicalType; +import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampNanosLogicalType; import io.trino.hive.formats.avro.model.AvroReadAction; import io.trino.hive.formats.avro.model.AvroReadAction.LongIoFunction; import io.trino.hive.formats.avro.model.BytesRead; @@ -33,6 +34,7 @@ import io.trino.spi.block.BlockBuilder; import io.trino.spi.type.DecimalType; import io.trino.spi.type.Int128; +import io.trino.spi.type.LongTimestamp; import io.trino.spi.type.Timestamps; import io.trino.spi.type.Type; import org.apache.avro.Schema; @@ -54,8 +56,14 @@ import static io.trino.spi.type.TimeType.TIME_MILLIS; import static io.trino.spi.type.TimestampType.TIMESTAMP_MICROS; import static io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS; +import static io.trino.spi.type.TimestampType.TIMESTAMP_NANOS; +import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND; +import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND; import static io.trino.spi.type.UuidType.UUID; import static io.trino.spi.type.UuidType.javaUuidToTrinoUuid; +import static java.lang.Math.divideExact; +import static java.lang.Math.floorMod; +import static java.lang.Math.multiplyExact; import static java.util.Objects.requireNonNull; /** @@ -136,6 +144,12 @@ static BlockBuildingDecoder getLogicalTypeBuildingFunction(AvroLogicalType logic } throw new IllegalStateException("Unreachable unfiltered logical type"); } + case TimestampNanosLogicalType _ -> { + if (readAction instanceof LongRead longRead) { + yield new TimestampNanosBlockBuildingDecoder(longRead); + } + throw new IllegalStateException("Unreachable unfiltered logical type"); + } case StringUUIDLogicalType _ -> { if (readAction instanceof StringRead) { yield new StringUUIDBlockBuildingDecoder(); @@ -271,6 +285,27 @@ public void decodeIntoBlock(Decoder decoder, BlockBuilder builder) } } + public static class TimestampNanosBlockBuildingDecoder + implements BlockBuildingDecoder + { + private final LongIoFunction longDecoder; + + public TimestampNanosBlockBuildingDecoder(LongRead longRead) + { + longDecoder = requireNonNull(longRead, "longRead is null").getLongDecoder(); + } + + @Override + public void decodeIntoBlock(Decoder decoder, BlockBuilder builder) + throws IOException + { + long epochNanos = longDecoder.apply(decoder); + long epochMicros = divideExact(epochNanos, NANOSECONDS_PER_MICROSECOND); + int picosOfMicro = multiplyExact(floorMod(epochNanos, NANOSECONDS_PER_MICROSECOND), PICOSECONDS_PER_NANOSECOND); + TIMESTAMP_NANOS.writeObject(builder, new LongTimestamp(epochMicros, picosOfMicro)); + } + } + public record StringUUIDBlockBuildingDecoder() implements BlockBuildingDecoder { diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java index 7df72f341966..42264e3e0e3b 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/NativeLogicalTypesAvroTypeManager.java @@ -25,6 +25,7 @@ import io.trino.hive.formats.avro.model.AvroLogicalType.TimeMillisLogicalType; import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampMicrosLogicalType; import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampMillisLogicalType; +import io.trino.hive.formats.avro.model.AvroLogicalType.TimestampNanosLogicalType; import io.trino.spi.block.Block; import io.trino.spi.type.DateType; import io.trino.spi.type.DecimalType; @@ -50,19 +51,27 @@ import java.util.function.BiFunction; import java.util.function.Function; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Verify.verify; import static io.trino.hive.formats.avro.model.AvroLogicalType.DATE; import static io.trino.hive.formats.avro.model.AvroLogicalType.DECIMAL; import static io.trino.hive.formats.avro.model.AvroLogicalType.LOCAL_TIMESTAMP_MICROS; import static io.trino.hive.formats.avro.model.AvroLogicalType.LOCAL_TIMESTAMP_MILLIS; +import static io.trino.hive.formats.avro.model.AvroLogicalType.LOCAL_TIMESTAMP_NANOS; import static io.trino.hive.formats.avro.model.AvroLogicalType.TIMESTAMP_MICROS; import static io.trino.hive.formats.avro.model.AvroLogicalType.TIMESTAMP_MILLIS; +import static io.trino.hive.formats.avro.model.AvroLogicalType.TIMESTAMP_NANOS; import static io.trino.hive.formats.avro.model.AvroLogicalType.TIME_MICROS; import static io.trino.hive.formats.avro.model.AvroLogicalType.TIME_MILLIS; import static io.trino.hive.formats.avro.model.AvroLogicalType.UUID; import static io.trino.hive.formats.avro.model.AvroLogicalType.fromAvroLogicalType; +import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND; +import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND; import static io.trino.spi.type.Timestamps.roundDiv; import static io.trino.spi.type.UuidType.trinoUuidToJavaUuid; +import static java.lang.Math.addExact; +import static java.lang.Math.divideExact; +import static java.lang.Math.multiplyExact; import static java.util.Objects.requireNonNull; import static org.apache.avro.LogicalTypes.fromSchemaIgnoreInvalid; @@ -79,6 +88,7 @@ public class NativeLogicalTypesAvroTypeManager public static final Schema TIME_MICROS_SCHEMA; public static final Schema TIMESTAMP_MILLIS_SCHEMA; public static final Schema TIMESTAMP_MICROS_SCHEMA; + public static final Schema TIMESTAMP_NANOS_SCHEMA; public static final Schema UUID_SCHEMA; static { @@ -92,6 +102,8 @@ public class NativeLogicalTypesAvroTypeManager LogicalTypes.timestampMillis().addToSchema(TIMESTAMP_MILLIS_SCHEMA); TIMESTAMP_MICROS_SCHEMA = SchemaBuilder.builder().longType(); LogicalTypes.timestampMicros().addToSchema(TIMESTAMP_MICROS_SCHEMA); + TIMESTAMP_NANOS_SCHEMA = SchemaBuilder.builder().longType(); + LogicalTypes.timestampNanos().addToSchema(TIMESTAMP_NANOS_SCHEMA); UUID_SCHEMA = Schema.create(Schema.Type.STRING); LogicalTypes.uuid().addToSchema(UUID_SCHEMA); } @@ -117,6 +129,7 @@ static Type getAvroLogicalTypeSpiType(AvroLogicalType avroLogicalType) case TimeMicrosLogicalType __ -> TimeType.TIME_MICROS; case TimestampMillisLogicalType __ -> TimestampType.TIMESTAMP_MILLIS; case TimestampMicrosLogicalType __ -> TimestampType.TIMESTAMP_MICROS; + case TimestampNanosLogicalType __ -> TimestampType.TIMESTAMP_NANOS; case StringUUIDLogicalType __ -> UuidType.UUID; }; } @@ -133,6 +146,7 @@ static Type getAvroLogicalTypeSpiType(LogicalType logicalType) case TIME_MICROS -> TimeType.TIME_MICROS; case TIMESTAMP_MILLIS -> TimestampType.TIMESTAMP_MILLIS; case TIMESTAMP_MICROS -> TimestampType.TIMESTAMP_MICROS; + case TIMESTAMP_NANOS -> TimestampType.TIMESTAMP_NANOS; case UUID -> UuidType.UUID; default -> throw new IllegalStateException("Unreachable unfiltered logical type"); }; @@ -216,6 +230,17 @@ private static BiFunction getAvroFunction(AvroLogicalTyp }; } } + case TimestampNanosLogicalType _ -> { + if (!(type instanceof TimestampType timestampType)) { + throw new AvroTypeException("Can't represent Avro logical type %s with Trino Type %s".formatted(logicalType, type)); + } + checkState(!timestampType.isShort(), "Short timestamp type cannot represent nanosecond precision"); + yield (block, position) -> + { + SqlTimestamp timestamp = ((SqlTimestamp) timestampType.getObjectValue(block, position)); + return addExact(multiplyExact(timestamp.getEpochMicros(), NANOSECONDS_PER_MICROSECOND), divideExact(timestamp.getPicosOfMicros(), PICOSECONDS_PER_NANOSECOND)); + }; + } case StringUUIDLogicalType _ -> { if (!(type instanceof UuidType uuidType)) { throw new AvroTypeException("Can't represent Avro logical type %s with Trino Type %s".formatted(logicalType, type)); @@ -253,10 +278,10 @@ public static ValidateLogicalTypeResult validateLogicalType(Schema schema) } LogicalType logicalType; switch (typeName) { - case DATE, DECIMAL, TIME_MILLIS, TIME_MICROS, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, UUID: + case DATE, DECIMAL, TIME_MILLIS, TIME_MICROS, TIMESTAMP_MILLIS, TIMESTAMP_MICROS, TIMESTAMP_NANOS, UUID: logicalType = fromSchemaIgnoreInvalid(schema); break; - case LOCAL_TIMESTAMP_MICROS, LOCAL_TIMESTAMP_MILLIS: + case LOCAL_TIMESTAMP_NANOS, LOCAL_TIMESTAMP_MICROS, LOCAL_TIMESTAMP_MILLIS: log.debug("Logical type %s not currently supported by by Trino", typeName); // fall through default: diff --git a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/model/AvroLogicalType.java b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/model/AvroLogicalType.java index 11c9327f0e1e..46476afef838 100644 --- a/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/model/AvroLogicalType.java +++ b/lib/trino-hive-formats/src/main/java/io/trino/hive/formats/avro/model/AvroLogicalType.java @@ -27,6 +27,7 @@ public sealed interface AvroLogicalType AvroLogicalType.TimeMicrosLogicalType, AvroLogicalType.TimestampMillisLogicalType, AvroLogicalType.TimestampMicrosLogicalType, + AvroLogicalType.TimestampNanosLogicalType, AvroLogicalType.StringUUIDLogicalType { // Copied from org.apache.avro.LogicalTypes @@ -36,8 +37,10 @@ public sealed interface AvroLogicalType String TIME_MICROS = "time-micros"; String TIMESTAMP_MILLIS = "timestamp-millis"; String TIMESTAMP_MICROS = "timestamp-micros"; + String TIMESTAMP_NANOS = "timestamp-nanos"; String LOCAL_TIMESTAMP_MILLIS = "local-timestamp-millis"; String LOCAL_TIMESTAMP_MICROS = "local-timestamp-micros"; + String LOCAL_TIMESTAMP_NANOS = "local-timestamp-nanos"; String UUID = "uuid"; static AvroLogicalType fromAvroLogicalType(LogicalType logicalType, Schema schema) @@ -80,6 +83,11 @@ static AvroLogicalType fromAvroLogicalType(LogicalType logicalType, Schema schem return new TimestampMicrosLogicalType(); } } + case TIMESTAMP_NANOS -> { + if (schema.getType() == Schema.Type.LONG) { + return new TimestampNanosLogicalType(); + } + } case UUID -> { if (schema.getType() == Schema.Type.STRING) { return new StringUUIDLogicalType(); @@ -110,6 +118,9 @@ record TimestampMillisLogicalType() record TimestampMicrosLogicalType() implements AvroLogicalType {} + record TimestampNanosLogicalType() + implements AvroLogicalType {} + record StringUUIDLogicalType() implements AvroLogicalType {} } diff --git a/lib/trino-hive-formats/src/test/java/io/trino/hive/formats/avro/TestAvroPageDataReaderWithAvroNativeTypeManagement.java b/lib/trino-hive-formats/src/test/java/io/trino/hive/formats/avro/TestAvroPageDataReaderWithAvroNativeTypeManagement.java index c9115444cbab..d48ecab15e8f 100644 --- a/lib/trino-hive-formats/src/test/java/io/trino/hive/formats/avro/TestAvroPageDataReaderWithAvroNativeTypeManagement.java +++ b/lib/trino-hive-formats/src/test/java/io/trino/hive/formats/avro/TestAvroPageDataReaderWithAvroNativeTypeManagement.java @@ -24,6 +24,7 @@ import io.trino.spi.type.DateType; import io.trino.spi.type.DecimalType; import io.trino.spi.type.Int128; +import io.trino.spi.type.LongTimestamp; import io.trino.spi.type.SqlDate; import io.trino.spi.type.SqlDecimal; import io.trino.spi.type.SqlTime; @@ -48,16 +49,21 @@ import java.util.Date; import java.util.UUID; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.DATE_SCHEMA; import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.TIMESTAMP_MICROS_SCHEMA; import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.TIMESTAMP_MILLIS_SCHEMA; +import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.TIMESTAMP_NANOS_SCHEMA; import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.TIME_MICROS_SCHEMA; import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.TIME_MILLIS_SCHEMA; import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.UUID_SCHEMA; import static io.trino.hive.formats.avro.NativeLogicalTypesAvroTypeManager.padBigEndianToSize; import static io.trino.spi.type.Decimals.MAX_SHORT_PRECISION; +import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_MILLISECOND; +import static java.lang.Math.floorDiv; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @TestInstance(PER_CLASS) @@ -101,6 +107,8 @@ public class TestAvroPageDataReaderWithAvroNativeTypeManagement .type(TIMESTAMP_MILLIS_SCHEMA).noDefault() .name("timestampMicros") .type(TIMESTAMP_MICROS_SCHEMA).noDefault() + .name("timestampNanos") + .type(TIMESTAMP_NANOS_SCHEMA).noDefault() .name("smallBytesDecimal") .type(DECIMAL_SMALL_BYTES_SCHEMA).noDefault() .name("smallFixedDecimal") @@ -124,14 +132,20 @@ public class TestAvroPageDataReaderWithAvroNativeTypeManagement ALL_SUPPORTED_TYPES_GENERIC_RECORD.put("timestampMillis", testTime.getTime()); BlockBuilder timestampMilliBlock = TimestampType.TIMESTAMP_MILLIS.createFixedSizeBlockBuilder(1); - TimestampType.TIMESTAMP_MILLIS.writeLong(timestampMilliBlock, testTime.getTime() * Timestamps.MICROSECONDS_PER_MILLISECOND); + TimestampType.TIMESTAMP_MILLIS.writeLong(timestampMilliBlock, testTime.getTime() * MICROSECONDS_PER_MILLISECOND); blocks.add(timestampMilliBlock.build()); - ALL_SUPPORTED_TYPES_GENERIC_RECORD.put("timestampMicros", testTime.getTime() * 1000); + ALL_SUPPORTED_TYPES_GENERIC_RECORD.put("timestampMicros", testTime.getTime() * MICROSECONDS_PER_MILLISECOND); BlockBuilder timestampMicroBlock = TimestampType.TIMESTAMP_MICROS.createFixedSizeBlockBuilder(1); - TimestampType.TIMESTAMP_MICROS.writeLong(timestampMicroBlock, testTime.getTime() * Timestamps.MICROSECONDS_PER_MILLISECOND); + TimestampType.TIMESTAMP_MICROS.writeLong(timestampMicroBlock, testTime.getTime() * MICROSECONDS_PER_MILLISECOND); blocks.add(timestampMicroBlock.build()); + ALL_SUPPORTED_TYPES_GENERIC_RECORD.put("timestampNanos", testTime.getTime() * Timestamps.NANOSECONDS_PER_MILLISECOND); + BlockBuilder timestampNanoBlock = TimestampType.TIMESTAMP_NANOS.createFixedSizeBlockBuilder(1); + SqlTimestamp sqlTimestamp = SqlTimestamp.fromMillis(9, testTime.getTime()); + TimestampType.TIMESTAMP_NANOS.writeObject(timestampNanoBlock, new LongTimestamp(sqlTimestamp.getEpochMicros(), sqlTimestamp.getPicosOfMicros())); + blocks.add(timestampNanoBlock.build()); + ALL_SUPPORTED_TYPES_GENERIC_RECORD.put("smallBytesDecimal", ByteBuffer.wrap(Longs.toByteArray(78068160000000L))); ALL_SUPPORTED_TYPES_GENERIC_RECORD.put("smallFixedDecimal", GENERIC_SMALL_FIXED_DECIMAL); BlockBuilder smallDecimalBlock = SMALL_DECIMAL_TYPE.createBlockBuilder(null, 1); @@ -265,20 +279,125 @@ public void testWriting() } } + @Test + public void testNegativeEpochTimestampsAndDates() + throws IOException, AvroTypeException + { + Location testLocation = createLocalTempLocation(); + Schema timestampsSchema = SchemaBuilder.builder() + .record("timestamps") + .fields() + .name("timestampMillis") + .type(TIMESTAMP_MILLIS_SCHEMA).noDefault() + .name("timestampMicros") + .type(TIMESTAMP_MICROS_SCHEMA).noDefault() + .name("timestampNanos") + .type(TIMESTAMP_NANOS_SCHEMA).noDefault() + .name("date") + .type(DATE_SCHEMA).noDefault() + .endRecord(); + ImmutableList.Builder blocks = ImmutableList.builder(); + + BlockBuilder timestampMillisBlock = TimestampType.TIMESTAMP_MILLIS.createFixedSizeBlockBuilder(2); + BlockBuilder timestampMicrosBlock = TimestampType.TIMESTAMP_MICROS.createFixedSizeBlockBuilder(2); + BlockBuilder timestampNanosBlock = TimestampType.TIMESTAMP_NANOS.createFixedSizeBlockBuilder(2); + + // negative epoch timestamps + long negativeEpochNanos = -1_000_000_000L; + SqlTimestamp negativeEpochTimestamp = SqlTimestamp.newInstance(9, negativeEpochNanos / Timestamps.NANOSECONDS_PER_MICROSECOND, 0); + { + TimestampType.TIMESTAMP_MILLIS.writeLong(timestampMillisBlock, negativeEpochNanos / Timestamps.NANOSECONDS_PER_MICROSECOND); + TimestampType.TIMESTAMP_MICROS.writeLong(timestampMicrosBlock, negativeEpochNanos / Timestamps.NANOSECONDS_PER_MICROSECOND); + TimestampType.TIMESTAMP_NANOS.writeObject(timestampNanosBlock, new LongTimestamp(negativeEpochTimestamp.getEpochMicros(), negativeEpochTimestamp.getPicosOfMicros())); + } + + // largest timestamp representable by nanos: 2262-04-11T23:47:16.854775807 + long largestEpochNanos = Long.MAX_VALUE; + SqlTimestamp largestEpochTimestamp = SqlTimestamp.newInstance(9, largestEpochNanos / Timestamps.NANOSECONDS_PER_MICROSECOND, 0); + { + TimestampType.TIMESTAMP_MILLIS.writeLong(timestampMillisBlock, largestEpochNanos / Timestamps.NANOSECONDS_PER_MICROSECOND); + TimestampType.TIMESTAMP_MICROS.writeLong(timestampMicrosBlock, largestEpochNanos / Timestamps.NANOSECONDS_PER_MICROSECOND); + TimestampType.TIMESTAMP_NANOS.writeObject(timestampNanosBlock, new LongTimestamp(largestEpochTimestamp.getEpochMicros(), largestEpochTimestamp.getPicosOfMicros())); + } + + blocks.add(timestampMillisBlock.build()); + blocks.add(timestampMicrosBlock.build()); + blocks.add(timestampNanosBlock.build()); + + BlockBuilder dateBlock = DateType.DATE.createFixedSizeBlockBuilder(2); + // negative epoch date + int daysBeforeEpoch = -3; + DateType.DATE.writeInt(dateBlock, daysBeforeEpoch); + + // largest timestamp representable by date: 5874897-07-11 + int daysAfterEpoch = Integer.MAX_VALUE; + DateType.DATE.writeInt(dateBlock, daysAfterEpoch); + + blocks.add(dateBlock.build()); + + Page timestampsPage = new Page(blocks.build().toArray(Block[]::new)); + try (AvroFileWriter fileWriter = new AvroFileWriter( + trinoLocalFilesystem.newOutputFile(testLocation).create(), + timestampsSchema, + new NativeLogicalTypesAvroTypeManager(), + AvroCompressionKind.NULL, + ImmutableMap.of(), + timestampsSchema.getFields().stream().map(Schema.Field::name).collect(toImmutableList()), + new NativeLogicalTypesAvroTypeBlockHandler().typeFor(timestampsSchema).getTypeParameters(), false)) { + fileWriter.write(timestampsPage); + } + + try (AvroFileReader fileReader = new AvroFileReader( + trinoLocalFilesystem.newInputFile(testLocation), + timestampsSchema, + new NativeLogicalTypesAvroTypeBlockHandler())) { + assertThat(fileReader.hasNext()).isTrue(); + Page p = fileReader.next(); + assertThat(fileReader.hasNext()).isFalse(); + assertThat(p.getPositionCount()).isEqualTo(2); + for (int i = 0; i < p.getPositionCount(); i++) { + SqlTimestamp readMillisTimestamp = (SqlTimestamp) TimestampType.TIMESTAMP_MILLIS.getObjectValue(p.getBlock(0), i); + SqlTimestamp readMicrosTimestamp = (SqlTimestamp) TimestampType.TIMESTAMP_MICROS.getObjectValue(p.getBlock(1), i); + SqlTimestamp readNanosTimestamp = (SqlTimestamp) TimestampType.TIMESTAMP_NANOS.getObjectValue(p.getBlock(2), i); + int readDaysValue = DateType.DATE.getInt(p.getBlock(3), i); + + if (i == 0) { + assertThat(readMillisTimestamp).isEqualTo(newTruncatedTimestamp(negativeEpochTimestamp, 3)); + assertThat(readMicrosTimestamp).isEqualTo(newTruncatedTimestamp(negativeEpochTimestamp, 6)); + assertThat(readNanosTimestamp).isEqualTo(negativeEpochTimestamp); + assertThat(readDaysValue).isEqualTo(daysBeforeEpoch); + } + else if (i == 1) { + assertThat(readMillisTimestamp).isEqualTo(newTruncatedTimestamp(largestEpochTimestamp, 3)); + assertThat(readMicrosTimestamp).isEqualTo(newTruncatedTimestamp(largestEpochTimestamp, 6)); + assertThat(readNanosTimestamp).isEqualTo(largestEpochTimestamp); + assertThat(readDaysValue).isEqualTo(daysAfterEpoch); + } + else { + fail("Expected only 2 positions"); + } + } + } + } + private static void assertIsAllSupportedTypePage(Page p) { assertThat(p.getPositionCount()).isEqualTo(1); // Timestamps equal SqlTimestamp milliTimestamp = (SqlTimestamp) TimestampType.TIMESTAMP_MILLIS.getObjectValue(p.getBlock(0), 0); SqlTimestamp microTimestamp = (SqlTimestamp) TimestampType.TIMESTAMP_MICROS.getObjectValue(p.getBlock(1), 0); + SqlTimestamp nanoTimestamp = (SqlTimestamp) TimestampType.TIMESTAMP_NANOS.getObjectValue(p.getBlock(2), 0); + assertThat(milliTimestamp).isEqualTo(microTimestamp.roundTo(3)); + assertThat(microTimestamp).isEqualTo(nanoTimestamp.roundTo(6)); assertThat(microTimestamp.getEpochMicros()).isEqualTo(testTime.getTime() * 1000); + assertThat(nanoTimestamp.getPicosOfMicros()).isEqualTo(0); // Decimals Equal - SqlDecimal smallBytesDecimal = (SqlDecimal) SMALL_DECIMAL_TYPE.getObjectValue(p.getBlock(2), 0); - SqlDecimal smallFixedDecimal = (SqlDecimal) SMALL_DECIMAL_TYPE.getObjectValue(p.getBlock(3), 0); - SqlDecimal largeBytesDecimal = (SqlDecimal) LARGE_DECIMAL_TYPE.getObjectValue(p.getBlock(4), 0); - SqlDecimal largeFixedDecimal = (SqlDecimal) LARGE_DECIMAL_TYPE.getObjectValue(p.getBlock(5), 0); + SqlDecimal smallBytesDecimal = (SqlDecimal) SMALL_DECIMAL_TYPE.getObjectValue(p.getBlock(3), 0); + SqlDecimal smallFixedDecimal = (SqlDecimal) SMALL_DECIMAL_TYPE.getObjectValue(p.getBlock(4), 0); + SqlDecimal largeBytesDecimal = (SqlDecimal) LARGE_DECIMAL_TYPE.getObjectValue(p.getBlock(5), 0); + SqlDecimal largeFixedDecimal = (SqlDecimal) LARGE_DECIMAL_TYPE.getObjectValue(p.getBlock(6), 0); assertThat(smallBytesDecimal).isEqualTo(smallFixedDecimal); assertThat(largeBytesDecimal).isEqualTo(largeFixedDecimal); @@ -286,16 +405,45 @@ private static void assertIsAllSupportedTypePage(Page p) assertThat(smallBytesDecimal.getUnscaledValue()).isEqualTo(new BigInteger(Longs.toByteArray(78068160000000L))); // Get date - SqlDate date = (SqlDate) DateType.DATE.getObjectValue(p.getBlock(6), 0); + SqlDate date = (SqlDate) DateType.DATE.getObjectValue(p.getBlock(7), 0); assertThat(date.getDays()).isEqualTo(9035); // Time equals - SqlTime timeMillis = (SqlTime) TimeType.TIME_MILLIS.getObjectValue(p.getBlock(7), 0); - SqlTime timeMicros = (SqlTime) TimeType.TIME_MICROS.getObjectValue(p.getBlock(8), 0); + SqlTime timeMillis = (SqlTime) TimeType.TIME_MILLIS.getObjectValue(p.getBlock(8), 0); + SqlTime timeMicros = (SqlTime) TimeType.TIME_MICROS.getObjectValue(p.getBlock(9), 0); assertThat(timeMillis).isEqualTo(timeMicros.roundTo(3)); assertThat(timeMillis.getPicos()).isEqualTo(timeMicros.getPicos()).isEqualTo(39_600_000_000L * 1_000_000L); //UUID - assertThat(RANDOM_UUID.toString()).isEqualTo(UuidType.UUID.getObjectValue(p.getBlock(9), 0)); + assertThat(RANDOM_UUID.toString()).isEqualTo(UuidType.UUID.getObjectValue(p.getBlock(10), 0)); + } + + /** + * Timestamps#rescale not exposed publicly + * + * @param timestamp timestamp to truncate + * @param truncateToPrecision precision of new truncated timestamp + * + * @return truncated timestamp + */ + private static SqlTimestamp newTruncatedTimestamp(SqlTimestamp timestamp, int truncateToPrecision) + { + checkArgument(truncateToPrecision >= 0 && truncateToPrecision <= 12, "Invalid precision: %s. Must be between 0 and 12.", truncateToPrecision); + + checkArgument(truncateToPrecision < timestamp.getPrecision(), + "Invalid truncateToPrecision %s. Must be less than timestamp precision %s.", + truncateToPrecision, + timestamp.getPrecision()); + + if (truncateToPrecision <= 6) { + long factor = (long) Math.pow(10, 6 - truncateToPrecision); + long truncatedEpochMicros = floorDiv(timestamp.getEpochMicros(), factor) * factor; + return SqlTimestamp.newInstance(truncateToPrecision, truncatedEpochMicros, 0); + } + else { + long factor = (long) Math.pow(10, 12 - truncateToPrecision); + long truncatedPicosOfMicros = floorDiv(timestamp.getPicosOfMicros(), factor) * factor; + return SqlTimestamp.newInstance(truncateToPrecision, timestamp.getEpochMicros(), (int) truncatedPicosOfMicros); + } } } diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaLiteral.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaLiteral.java index 1b24eff6a648..2877f5015f26 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaLiteral.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/hive/TestAvroSchemaLiteral.java @@ -13,14 +13,22 @@ */ package io.trino.tests.product.hive; +import com.google.common.collect.ImmutableList; +import io.trino.plugin.hive.HiveTimestampPrecision; +import io.trino.tempto.query.QueryExecutionException; import org.intellij.lang.annotations.Language; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.util.Locale; +import java.util.Optional; + import static io.trino.tempto.assertions.QueryAssert.Row.row; import static io.trino.tests.product.utils.QueryExecutors.onHive; import static io.trino.tests.product.utils.QueryExecutors.onTrino; import static java.lang.String.format; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class TestAvroSchemaLiteral extends HiveProductTest @@ -71,4 +79,79 @@ public void testTrinoCreatedTable() onTrino().executeQuery("DROP TABLE test_avro_schema_literal_trino"); } + + @Test(dataProvider = "timestampPrecisionTestCases") + public void testTimestampPrecisionMapping(TimestampPrecisionTestCase timestampPrecisionTestCase) + { + HiveTimestampPrecision sessionTimestampPrecision = timestampPrecisionTestCase.sessionTimestampPrecision(); + onTrino().executeQuery(format("DROP TABLE IF EXISTS %s", timestampPrecisionTestCase.tableName())); + onTrino().executeQuery(format("SET SESSION hive.timestamp_precision = '%s'", sessionTimestampPrecision.name())); + + String createTableSql = format("CREATE TABLE %s (%s %s) WITH (format='AVRO', avro_schema_literal='%s')", + timestampPrecisionTestCase.tableName(), + timestampPrecisionTestCase.trinoColumnName(), + timestampPrecisionTestCase.trinoCreateType(), + format(""" + { + "namespace": "io.trino.test", + "name": "product_tests_avro_table_%s", + "type": "record", + "fields": [ + { "name":"%s", "type":{"type":"long","logicalType":"%s"} } + ] + }""", timestampPrecisionTestCase.trinoColumnName(), timestampPrecisionTestCase.trinoColumnName(), timestampPrecisionTestCase.avroLogicalType())); + if (timestampPrecisionTestCase.trinoReadType().isEmpty()) { + assertThatThrownBy(() -> onTrino().executeQuery(createTableSql)) + .isInstanceOf(QueryExecutionException.class) + .hasMessageContaining("Incorrect timestamp precision for %s; the configured precision is %s; column name: %s".formatted(timestampPrecisionTestCase.trinoCreateType().toLowerCase(Locale.ENGLISH), sessionTimestampPrecision.name(), timestampPrecisionTestCase.trinoColumnName())); + return; + } + + onTrino().executeQuery(createTableSql); + assertThat(onTrino().executeQuery(format("SHOW COLUMNS FROM %s", timestampPrecisionTestCase.tableName()))) + .containsExactlyInOrder(row(timestampPrecisionTestCase.trinoColumnName(), timestampPrecisionTestCase.trinoReadType().get().toLowerCase(Locale.ENGLISH), "", "")); + } + + @DataProvider + public TimestampPrecisionTestCase[] timestampPrecisionTestCases() + { + ImmutableList.Builder testCases = ImmutableList.builder(); + HiveTimestampPrecision[] sessionPrecisionValues = assertThat(HiveTimestampPrecision.values()) + .hasSize(3) + .actual(); + for (HiveTimestampPrecision hiveTimestampPrecision : sessionPrecisionValues) { + testCases.add(new TimestampPrecisionTestCase( + hiveTimestampPrecision, + "timestamp-millis", + "ts_millis", + "TIMESTAMP(3)", + hiveTimestampPrecision.equals(HiveTimestampPrecision.MILLISECONDS) ? Optional.of("TIMESTAMP(3)") : Optional.empty())); + testCases.add(new TimestampPrecisionTestCase( + hiveTimestampPrecision, + "timestamp-micros", + "ts_micros", + "TIMESTAMP(6)", + hiveTimestampPrecision.equals(HiveTimestampPrecision.MICROSECONDS) ? Optional.of("BIGINT") : Optional.empty())); + testCases.add(new TimestampPrecisionTestCase( + hiveTimestampPrecision, + "timestamp-nanos", + "ts_nanos", + "TIMESTAMP(9)", + hiveTimestampPrecision.equals(HiveTimestampPrecision.NANOSECONDS) ? Optional.of("BIGINT") : Optional.empty())); + } + return testCases.build().toArray(TimestampPrecisionTestCase[]::new); + } + + public record TimestampPrecisionTestCase( + HiveTimestampPrecision sessionTimestampPrecision, + String avroLogicalType, + String trinoColumnName, + String trinoCreateType, + Optional trinoReadType) + { + public String tableName() + { + return format("test_avro_schema_literal_trino_%s_%s", trinoColumnName, sessionTimestampPrecision.name()); + } + } } From c880edcd1d0dc71d24bf77f0fd37d6ef3f7cc7c3 Mon Sep 17 00:00:00 2001 From: Grant Nicholas Date: Tue, 9 Dec 2025 13:58:10 -0600 Subject: [PATCH 2/2] CI