From 26c4e4f1a9dcfe01cd6511a574ed903c17863cab Mon Sep 17 00:00:00 2001 From: vinodkc Date: Sat, 6 Dec 2025 15:07:59 -0800 Subject: [PATCH 1/5] Add codegen for TIME numeric conversion functions --- .../expressions/timeExpressions.scala | 119 +++++++++++++----- 1 file changed, 85 insertions(+), 34 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 8796bf27a33c..018e31163b90 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -34,22 +34,11 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.catalyst.util.TimeFormatter import org.apache.spark.sql.catalyst.util.TypeUtils.ordinalNumber import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.types.StringTypeWithCollation import org.apache.spark.sql.types.{AbstractDataType, AnyTimeType, ByteType, DataType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, IntegralType, LongType, NumericType, ObjectType, TimeType} import org.apache.spark.sql.types.DayTimeIntervalType.{HOUR, SECOND} import org.apache.spark.unsafe.types.UTF8String -trait TimeExpression extends Expression { - override def checkInputDataTypes(): TypeCheckResult = { - if (SQLConf.get.isTimeTypeEnabled) { - super.checkInputDataTypes() - } else { - throw QueryCompilationErrors.unsupportedTimeTypeError() - } - } -} - /** * Parses a column to a time based on the given format. */ @@ -77,7 +66,7 @@ trait TimeExpression extends Expression { since = "4.1.0") // scalastyle:on line.size.limit case class ToTime(str: Expression, format: Option[Expression]) - extends RuntimeReplaceable with ExpectsInputTypes with TimeExpression { + extends RuntimeReplaceable with ExpectsInputTypes { def this(str: Expression, format: Expression) = this(str, Option(format)) def this(str: Expression) = this(str, None) @@ -213,7 +202,7 @@ object TryToTimeExpressionBuilder extends ExpressionBuilder { // scalastyle:on line.size.limit case class MinutesOfTime(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes with TimeExpression { + with ExpectsInputTypes { override def replacement: Expression = StaticInvoke( classOf[DateTimeUtils.type], @@ -272,7 +261,7 @@ object MinuteExpressionBuilder extends ExpressionBuilder { case class HoursOfTime(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes with TimeExpression { + with ExpectsInputTypes { override def replacement: Expression = StaticInvoke( classOf[DateTimeUtils.type], @@ -329,7 +318,7 @@ object HourExpressionBuilder extends ExpressionBuilder { case class SecondsOfTimeWithFraction(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes with TimeExpression { + with ExpectsInputTypes { override def replacement: Expression = { val precision = child.dataType match { case TimeType(p) => p @@ -355,7 +344,7 @@ case class SecondsOfTimeWithFraction(child: Expression) case class SecondsOfTime(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes with TimeExpression { + with ExpectsInputTypes { override def replacement: Expression = StaticInvoke( classOf[DateTimeUtils.type], @@ -446,8 +435,7 @@ object SecondExpressionBuilder extends ExpressionBuilder { case class CurrentTime( child: Expression = Literal(TimeType.MICROS_PRECISION), timeZoneId: Option[String] = None) extends UnaryExpression - with TimeZoneAwareExpression with ImplicitCastInputTypes with CodegenFallback - with TimeExpression { + with TimeZoneAwareExpression with ImplicitCastInputTypes with CodegenFallback { def this() = { this(Literal(TimeType.MICROS_PRECISION), None) @@ -559,7 +547,7 @@ case class MakeTime( secsAndMicros: Expression) extends RuntimeReplaceable with ImplicitCastInputTypes - with ExpectsInputTypes with TimeExpression { + with ExpectsInputTypes { // Accept `sec` as DecimalType to avoid loosing precision of microseconds while converting // it to the fractional part of `sec`. If `sec` is an IntegerType, it can be cast into decimal @@ -584,8 +572,7 @@ case class MakeTime( * Adds day-time interval to time. */ case class TimeAddInterval(time: Expression, interval: Expression) - extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes - with TimeExpression { + extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes { override def nullIntolerant: Boolean = true override def left: Expression = time @@ -626,8 +613,7 @@ case class TimeAddInterval(time: Expression, interval: Expression) * Returns a day-time interval between time values. */ case class SubtractTimes(left: Expression, right: Expression) - extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes - with TimeExpression { + extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes { override def nullIntolerant: Boolean = true override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType, AnyTimeType) @@ -684,8 +670,7 @@ case class TimeDiff( end: Expression) extends TernaryExpression with RuntimeReplaceable - with ImplicitCastInputTypes - with TimeExpression { + with ImplicitCastInputTypes { override def first: Expression = unit override def second: Expression = start @@ -740,8 +725,7 @@ case class TimeDiff( since = "4.1.0") // scalastyle:on line.size.limit case class TimeTrunc(unit: Expression, time: Expression) - extends BinaryExpression with RuntimeReplaceable with ImplicitCastInputTypes - with TimeExpression { + extends BinaryExpression with RuntimeReplaceable with ImplicitCastInputTypes { override def left: Expression = unit override def right: Expression = time @@ -769,8 +753,7 @@ case class TimeTrunc(unit: Expression, time: Expression) } abstract class IntegralToTimeBase - extends UnaryExpression with ExpectsInputTypes with CodegenFallback - with TimeExpression { + extends UnaryExpression with ExpectsInputTypes { protected def upScaleFactor: Long override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) @@ -787,10 +770,25 @@ abstract class IntegralToTimeBase val nanos = Math.multiplyExact(input.asInstanceOf[Number].longValue(), upScaleFactor) validateTimeNanos(nanos) } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + nullSafeCodeGen(ctx, ev, c => { + val nanos = if (upScaleFactor == 1) { + c + } else { + s"java.lang.Math.multiplyExact($c, ${upScaleFactor}L)" + } + s""" + |${ev.value} = $nanos; + |if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { + | ${ev.isNull} = true; + |} + |""".stripMargin + }) + } } -abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes - with TimeExpression { +abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes { protected def scaleFactor: Long override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) @@ -837,8 +835,7 @@ abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeFromSeconds(child: Expression) - extends UnaryExpression with ExpectsInputTypes with CodegenFallback - with TimeExpression { + extends UnaryExpression with ExpectsInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(NumericType) override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) override def nullable: Boolean = true @@ -878,6 +875,46 @@ case class TimeFromSeconds(child: Expression) override def nullSafeEval(input: Any): Any = evalFunc(input) + override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = child.dataType match { + case _: IntegralType => + nullSafeCodeGen(ctx, ev, c => { + val nanos = s"java.lang.Math.multiplyExact($c, ${NANOS_PER_SECOND}L)" + s""" + |${ev.value} = $nanos; + |if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { + | ${ev.isNull} = true; + |} + |""".stripMargin + }) + case _: DecimalType => + val bd = ctx.addReferenceObj("nanoSecondFactor", + new java.math.BigDecimal(NANOS_PER_SECOND), + classOf[java.math.BigDecimal].getName) + nullSafeCodeGen(ctx, ev, c => { + s""" + |${ev.value} = $c.toJavaBigDecimal().multiply($bd).longValueExact(); + |if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { + | ${ev.isNull} = true; + |} + |""".stripMargin + }) + case other => + val castToDouble = if (other.isInstanceOf[FloatType]) "(double)" else "" + nullSafeCodeGen(ctx, ev, c => { + val typeStr = CodeGenerator.boxedType(other) + s""" + |if ($typeStr.isNaN($c) || $typeStr.isInfinite($c)) { + | ${ev.isNull} = true; + |} else { + | ${ev.value} = (long)($castToDouble$c * ${NANOS_PER_SECOND}L); + | if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { + | ${ev.isNull} = true; + | } + |} + |""".stripMargin + }) + } + override def prettyName: String = "time_from_seconds" override protected def withNewChildInternal(newChild: Expression): TimeFromSeconds = @@ -971,7 +1008,7 @@ case class TimeFromMicros(child: Expression) group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeToSeconds(child: Expression) - extends UnaryExpression with ImplicitCastInputTypes with CodegenFallback { + extends UnaryExpression with ImplicitCastInputTypes { override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) override def dataType: DataType = DecimalType(14, 6) @@ -984,6 +1021,20 @@ case class TimeToSeconds(child: Expression) if (result.changePrecision(14, 6)) result else null } + override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val divisor = ctx.addReferenceObj("nanoSecondDecimal", + Decimal(NANOS_PER_SECOND), + "org.apache.spark.sql.types.Decimal") + nullSafeCodeGen(ctx, ev, nanos => { + s""" + |${ev.value} = org.apache.spark.sql.types.Decimal.apply($nanos).$$div($divisor); + |if (!${ev.value}.changePrecision(14, 6)) { + | ${ev.isNull} = true; + |} + |""".stripMargin + }) + } + override def prettyName: String = "time_to_seconds" override protected def withNewChildInternal(newChild: Expression): TimeToSeconds = From 64f386bf059417443fc5598bf6fc2ce2682d7f83 Mon Sep 17 00:00:00 2001 From: vinodkc Date: Mon, 8 Dec 2025 17:55:03 -0800 Subject: [PATCH 2/5] Handled ANSI mode and refactored to unify the interpreted and codegen code paths --- .../expressions/timeExpressions.scala | 243 +++++----- .../sql/catalyst/util/DateTimeUtils.scala | 106 +++- .../expressions/TimeExpressionsSuite.scala | 85 +++- .../sql-tests/analyzer-results/time.sql.out | 348 +++++++++++++ .../test/resources/sql-tests/inputs/time.sql | 93 +++- .../resources/sql-tests/results/time.sql.out | 456 ++++++++++++++++++ 6 files changed, 1180 insertions(+), 151 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 018e31163b90..89e546701783 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -34,11 +34,83 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.catalyst.util.TimeFormatter import org.apache.spark.sql.catalyst.util.TypeUtils.ordinalNumber import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.types.StringTypeWithCollation -import org.apache.spark.sql.types.{AbstractDataType, AnyTimeType, ByteType, DataType, DayTimeIntervalType, Decimal, DecimalType, DoubleType, FloatType, IntegerType, IntegralType, LongType, NumericType, ObjectType, TimeType} +import org.apache.spark.sql.types.{AbstractDataType, AnyTimeType, ByteType, DataType, DayTimeIntervalType, DecimalType, IntegerType, IntegralType, LongType, NumericType, ObjectType, TimeType} import org.apache.spark.sql.types.DayTimeIntervalType.{HOUR, SECOND} import org.apache.spark.unsafe.types.UTF8String +/** + * Helper trait for TIME conversion expressions with consistent error handling. + */ +trait TimeConversionErrorHandling { + def failOnError: Boolean + + /** Wraps evaluation with error handling (throws in ANSI mode, null otherwise). */ + protected def evalWithErrorHandling[T](f: => T): Any = { + try { + f + } catch { + case e: DateTimeException if failOnError => + throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e) + case e: ArithmeticException if failOnError => + throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange( + new DateTimeException(s"Overflow in TIME conversion: ${e.getMessage}")) + case _: DateTimeException | _: ArithmeticException => null + } + } + + /** Generates error handling code (DateTimeException + ArithmeticException). */ + protected def doGenErrorHandling( + ctx: CodegenContext, + ev: ExprCode, + utilCall: String): String = { + val dateTimeErrorBranch = if (failOnError) { + "throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e);" + } else { + s"${ev.isNull} = true;" + } + + val arithmeticErrorBranch = if (failOnError) { + s""" + |throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange( + | new java.time.DateTimeException("Overflow in TIME conversion: " + e.getMessage())); + |""".stripMargin + } else { + s"${ev.isNull} = true;" + } + + s""" + |try { + | ${ev.value} = $utilCall; + |} catch (java.time.DateTimeException e) { + | $dateTimeErrorBranch + |} catch (java.lang.ArithmeticException e) { + | $arithmeticErrorBranch + |} + |""".stripMargin + } + + /** Generates error handling code (DateTimeException only). */ + protected def doGenDateTimeError( + ctx: CodegenContext, + ev: ExprCode, + utilCall: String): String = { + val errorBranch = if (failOnError) { + "throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e);" + } else { + s"${ev.isNull} = true;" + } + s""" + |try { + | ${ev.value} = $utilCall; + |} catch (java.time.DateTimeException e) { + | $errorBranch + |} + |""".stripMargin + } +} + /** * Parses a column to a time based on the given format. */ @@ -753,59 +825,33 @@ case class TimeTrunc(unit: Expression, time: Expression) } abstract class IntegralToTimeBase - extends UnaryExpression with ExpectsInputTypes { + extends UnaryExpression with ExpectsInputTypes with TimeConversionErrorHandling { protected def upScaleFactor: Long + def failOnError: Boolean = SQLConf.get.ansiEnabled override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) override def nullable: Boolean = true override def nullIntolerant: Boolean = true - @inline - protected final def validateTimeNanos(nanos: Long): Any = { - if (nanos < 0 || nanos >= NANOS_PER_DAY) null else nanos - } - override protected def nullSafeEval(input: Any): Any = { - val nanos = Math.multiplyExact(input.asInstanceOf[Number].longValue(), upScaleFactor) - validateTimeNanos(nanos) + evalWithErrorHandling { + DateTimeUtils.timeFromIntegral(input.asInstanceOf[Number].longValue(), upScaleFactor) + } } override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - nullSafeCodeGen(ctx, ev, c => { - val nanos = if (upScaleFactor == 1) { - c - } else { - s"java.lang.Math.multiplyExact($c, ${upScaleFactor}L)" - } - s""" - |${ev.value} = $nanos; - |if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { - | ${ev.isNull} = true; - |} - |""".stripMargin - }) + val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") + nullSafeCodeGen(ctx, ev, c => + doGenErrorHandling(ctx, ev, s"$dtu.timeFromIntegral($c, ${upScaleFactor}L)") + ) } } abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes { - protected def scaleFactor: Long - override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) override def dataType: DataType = LongType override def nullIntolerant: Boolean = true - - override def nullSafeEval(input: Any): Any = { - Math.floorDiv(input.asInstanceOf[Number].longValue(), scaleFactor) - } - - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - if (scaleFactor == 1) { - defineCodeGen(ctx, ev, c => c) - } else { - defineCodeGen(ctx, ev, c => s"java.lang.Math.floorDiv($c, ${scaleFactor}L)") - } - } } // scalastyle:off line.size.limit @@ -835,84 +881,26 @@ abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes { group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeFromSeconds(child: Expression) - extends UnaryExpression with ExpectsInputTypes { + extends UnaryExpression with ExpectsInputTypes with TimeConversionErrorHandling { override def inputTypes: Seq[AbstractDataType] = Seq(NumericType) override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) override def nullable: Boolean = true override def nullIntolerant: Boolean = true - @inline - private def validateTimeNanos(nanos: Long): Any = { - if (nanos < 0 || nanos >= NANOS_PER_DAY) null else nanos - } + def failOnError: Boolean = SQLConf.get.ansiEnabled - @transient - private lazy val evalFunc: Any => Any = child.dataType match { - case _: IntegralType => input => - val nanos = Math.multiplyExact(input.asInstanceOf[Number].longValue(), NANOS_PER_SECOND) - validateTimeNanos(nanos) - case _: DecimalType => input => - val operand = new java.math.BigDecimal(NANOS_PER_SECOND) - val nanos = input.asInstanceOf[Decimal].toJavaBigDecimal.multiply(operand).longValueExact() - validateTimeNanos(nanos) - case _: FloatType => input => - val f = input.asInstanceOf[Float] - if (f.isNaN || f.isInfinite) { - null - } else { - val nanos = (f.toDouble * NANOS_PER_SECOND).toLong - validateTimeNanos(nanos) - } - case _: DoubleType => input => - val d = input.asInstanceOf[Double] - if (d.isNaN || d.isInfinite) { - null - } else { - val nanos = (d * NANOS_PER_SECOND).toLong - validateTimeNanos(nanos) - } + override def nullSafeEval(input: Any): Any = { + evalWithErrorHandling { + DateTimeUtils.timeFromSeconds(input, child.dataType) + } } - override def nullSafeEval(input: Any): Any = evalFunc(input) - - override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = child.dataType match { - case _: IntegralType => - nullSafeCodeGen(ctx, ev, c => { - val nanos = s"java.lang.Math.multiplyExact($c, ${NANOS_PER_SECOND}L)" - s""" - |${ev.value} = $nanos; - |if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { - | ${ev.isNull} = true; - |} - |""".stripMargin - }) - case _: DecimalType => - val bd = ctx.addReferenceObj("nanoSecondFactor", - new java.math.BigDecimal(NANOS_PER_SECOND), - classOf[java.math.BigDecimal].getName) - nullSafeCodeGen(ctx, ev, c => { - s""" - |${ev.value} = $c.toJavaBigDecimal().multiply($bd).longValueExact(); - |if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { - | ${ev.isNull} = true; - |} - |""".stripMargin - }) - case other => - val castToDouble = if (other.isInstanceOf[FloatType]) "(double)" else "" - nullSafeCodeGen(ctx, ev, c => { - val typeStr = CodeGenerator.boxedType(other) - s""" - |if ($typeStr.isNaN($c) || $typeStr.isInfinite($c)) { - | ${ev.isNull} = true; - |} else { - | ${ev.value} = (long)($castToDouble$c * ${NANOS_PER_SECOND}L); - | if (${ev.value} < 0L || ${ev.value} >= ${NANOS_PER_DAY}L) { - | ${ev.isNull} = true; - | } - |} - |""".stripMargin - }) + override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") + val dt = ctx.addReferenceObj("childDataType", child.dataType) + nullSafeCodeGen(ctx, ev, c => + doGenErrorHandling(ctx, ev, s"$dtu.timeFromSeconds($c, $dt)") + ) } override def prettyName: String = "time_from_seconds" @@ -1008,31 +996,26 @@ case class TimeFromMicros(child: Expression) group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeToSeconds(child: Expression) - extends UnaryExpression with ImplicitCastInputTypes { + extends UnaryExpression with ImplicitCastInputTypes with TimeConversionErrorHandling { override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) override def dataType: DataType = DecimalType(14, 6) override def nullable: Boolean = true override def nullIntolerant: Boolean = true + def failOnError: Boolean = SQLConf.get.ansiEnabled + protected override def nullSafeEval(input: Any): Any = { - val nanos = input.asInstanceOf[Long] - val result = Decimal(nanos) / Decimal(NANOS_PER_SECOND) - if (result.changePrecision(14, 6)) result else null + evalWithErrorHandling { + DateTimeUtils.timeToSeconds(input.asInstanceOf[Long]) + } } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - val divisor = ctx.addReferenceObj("nanoSecondDecimal", - Decimal(NANOS_PER_SECOND), - "org.apache.spark.sql.types.Decimal") - nullSafeCodeGen(ctx, ev, nanos => { - s""" - |${ev.value} = org.apache.spark.sql.types.Decimal.apply($nanos).$$div($divisor); - |if (!${ev.value}.changePrecision(14, 6)) { - | ${ev.isNull} = true; - |} - |""".stripMargin - }) + val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") + nullSafeCodeGen(ctx, ev, nanos => + doGenDateTimeError(ctx, ev, s"$dtu.timeToSeconds($nanos)") + ) } override def prettyName: String = "time_to_seconds" @@ -1066,7 +1049,14 @@ case class TimeToSeconds(child: Expression) case class TimeToMillis(child: Expression) extends TimeToLongBase { - override def scaleFactor: Long = NANOS_PER_MILLIS + override def nullSafeEval(input: Any): Any = { + DateTimeUtils.timeToMillis(input.asInstanceOf[Number].longValue()) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") + defineCodeGen(ctx, ev, c => s"$dtu.timeToMillis($c)") + } override def prettyName: String = "time_to_millis" @@ -1099,7 +1089,14 @@ case class TimeToMillis(child: Expression) case class TimeToMicros(child: Expression) extends TimeToLongBase { - override def scaleFactor: Long = NANOS_PER_MICROS + override def nullSafeEval(input: Any): Any = { + DateTimeUtils.timeToMicros(input.asInstanceOf[Number].longValue()) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") + defineCodeGen(ctx, ev, c => s"$dtu.timeToMicros($c)") + } override def prettyName: String = "time_to_micros" diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 9310f4b9ae75..2017ad9de792 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -28,7 +28,7 @@ import scala.util.control.NonFatal import org.apache.spark.{QueryContext, SparkException, SparkIllegalArgumentException} import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.types.{Decimal, DoubleExactNumeric, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{DataType, Decimal, DecimalType, DoubleExactNumeric, DoubleType, FloatType, IntegralType, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} /** @@ -897,6 +897,110 @@ object DateTimeUtils extends SparkDateTimeUtils { } } + /** + * Creates a TIME value from seconds since midnight. + * @param seconds Numeric value (0 to 86399.999999) + * @param dataType Input data type + * @return Nanoseconds since midnight + */ + def timeFromSeconds(seconds: Any, dataType: DataType): Long = { + val nanos = dataType match { + case _: IntegralType => + Math.multiplyExact(seconds.asInstanceOf[Number].longValue(), NANOS_PER_SECOND) + case _: DecimalType => + val operand = new java.math.BigDecimal(NANOS_PER_SECOND) + seconds.asInstanceOf[Decimal].toJavaBigDecimal.multiply(operand).longValueExact() + case FloatType => + val f = seconds.asInstanceOf[Float] + if (f.isNaN || f.isInfinite) { + throw new DateTimeException("Cannot convert NaN or Infinite value to TIME") + } + (f.toDouble * NANOS_PER_SECOND).toLong + case DoubleType => + val d = seconds.asInstanceOf[Double] + if (d.isNaN || d.isInfinite) { + throw new DateTimeException("Cannot convert NaN or Infinite value to TIME") + } + (d * NANOS_PER_SECOND).toLong + } + validateTimeNanos(nanos) + } + + /** + * Creates a TIME value from milliseconds since midnight. + * @param millis Milliseconds (0 to 86399999) + * @return Nanoseconds since midnight + */ + def timeFromMillis(millis: Long): Long = { + timeFromIntegral(millis, NANOS_PER_MILLIS) + } + + /** + * Creates a TIME value from microseconds since midnight. + * @param micros Microseconds (0 to 86399999999) + * @return Nanoseconds since midnight + */ + def timeFromMicros(micros: Long): Long = { + timeFromIntegral(micros, NANOS_PER_MICROS) + } + + /** + * Creates a TIME value from an integral value with a scale factor. + * @param value Integral value + * @param scaleFactor Conversion factor to nanoseconds + * @return Nanoseconds since midnight + */ + def timeFromIntegral(value: Long, scaleFactor: Long): Long = { + val nanos = if (scaleFactor == 1) value else Math.multiplyExact(value, scaleFactor) + validateTimeNanos(nanos) + } + + /** + * Validates nanoseconds is within valid TIME range [0, NANOS_PER_DAY). + * @param nanos Nanoseconds to validate + * @return Input nanos if valid + */ + private def validateTimeNanos(nanos: Long): Long = { + if (nanos < 0 || nanos >= NANOS_PER_DAY) { + throw new DateTimeException( + s"Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, " + + s"but got $nanos nanoseconds") + } + nanos + } + + /** + * Converts a TIME value to seconds. + * @param nanos Nanoseconds since midnight + * @return Seconds as Decimal(14, 6) + */ + def timeToSeconds(nanos: Long): Decimal = { + val result = Decimal(nanos) / Decimal(NANOS_PER_SECOND) + if (!result.changePrecision(14, 6)) { + throw new DateTimeException( + "TIME to seconds conversion resulted in value that cannot fit in Decimal(14, 6)") + } + result + } + + /** + * Converts a TIME value to milliseconds. + * @param nanos Nanoseconds since midnight + * @return Milliseconds since midnight + */ + def timeToMillis(nanos: Long): Long = { + Math.floorDiv(nanos, NANOS_PER_MILLIS) + } + + /** + * Converts a TIME value to microseconds. + * @param nanos Nanoseconds since midnight + * @return Microseconds since midnight + */ + def timeToMicros(nanos: Long): Long = { + Math.floorDiv(nanos, NANOS_PER_MICROS) + } + /** * Makes a timestamp without time zone from a date and a local time. * diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala index a4d46f77f648..a1e92e959432 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala @@ -25,6 +25,7 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLId, toSQLValue} import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ import org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.localTimeToNanos +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, IntegerType, LongType, StringType, TimeType} import org.apache.spark.sql.types.DayTimeIntervalType.{DAY, HOUR, SECOND} @@ -593,20 +594,76 @@ class TimeExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } test("Numeric to TIME conversions - range validation") { - // time_from_seconds - out of range [0, 86400) - checkEvaluation(TimeFromSeconds(Literal(-1L)), null) - checkEvaluation(TimeFromSeconds(Literal(86400L)), null) - checkEvaluation(TimeFromSeconds(Literal(90000L)), null) - checkEvaluation(TimeFromSeconds(Literal(Decimal(-0.1))), null) - checkEvaluation(TimeFromSeconds(Literal(Decimal(86400.0))), null) - - // time_from_millis - out of range [0, 86400000) - checkEvaluation(TimeFromMillis(Literal(-1L)), null) - checkEvaluation(TimeFromMillis(Literal(86400000L)), null) - - // time_from_micros - out of range [0, 86400000000) - checkEvaluation(TimeFromMicros(Literal(-1L)), null) - checkEvaluation(TimeFromMicros(Literal(86400000000L)), null) + // Test non-ANSI mode: returns NULL for out-of-range values + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + // time_from_seconds - out of range [0, 86400) + checkEvaluation(TimeFromSeconds(Literal(-1L)), null) + checkEvaluation(TimeFromSeconds(Literal(86400L)), null) + checkEvaluation(TimeFromSeconds(Literal(90000L)), null) + checkEvaluation(TimeFromSeconds(Literal(Decimal(-0.1))), null) + checkEvaluation(TimeFromSeconds(Literal(Decimal(86400.0))), null) + + // time_from_millis - out of range [0, 86400000) + checkEvaluation(TimeFromMillis(Literal(-1L)), null) + checkEvaluation(TimeFromMillis(Literal(86400000L)), null) + + // time_from_micros - out of range [0, 86400000000) + checkEvaluation(TimeFromMicros(Literal(-1L)), null) + checkEvaluation(TimeFromMicros(Literal(86400000000L)), null) + + // Test overflow in TIME conversion - returns NULL in non-ANSI mode + checkEvaluation(TimeFromSeconds(Literal(Long.MaxValue)), null) + + // Test NaN and Infinite for floating point - returns NULL in non-ANSI mode + checkEvaluation(TimeFromSeconds(Literal(Float.NaN)), null) + checkEvaluation(TimeFromSeconds(Literal(Double.PositiveInfinity)), null) + } + + // Test ANSI mode: throws exceptions for out-of-range values + withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { + // time_from_seconds - out of range [0, 86400) + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(-1L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(86400L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Decimal(-0.1))), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Decimal(86400.0))), + "Invalid TIME value") + + // time_from_millis - out of range [0, 86400000) + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMillis(Literal(-1L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMillis(Literal(86400000L)), + "Invalid TIME value") + + // time_from_micros - out of range [0, 86400000000) + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMicros(Literal(-1L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMicros(Literal(86400000000L)), + "Invalid TIME value") + + // Test overflow in TIME conversion + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Long.MaxValue)), + "Overflow in TIME conversion") + + // Test NaN and Infinite for floating point + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Float.NaN)), + "Cannot convert NaN or Infinite value to TIME") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Double.PositiveInfinity)), + "Cannot convert NaN or Infinite value to TIME") + } } test("Numeric to TIME conversions - NULL inputs") { diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out index 7075a9f8c4b4..bcf0c4cb6da5 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out @@ -1960,6 +1960,354 @@ Project [cast(00:00:00.1234 as time(4)) - 23:59:59 AS CAST(00:00:00.1234 AS TIME +- OneRowRelation +-- !query +SET spark.sql.ansi.enabled = false +-- !query analysis +SetCommand (spark.sql.ansi.enabled,Some(false)) + + +-- !query +SELECT time_from_seconds(0) +-- !query analysis +Project [time_from_seconds(0) AS time_from_seconds(0)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(43200) +-- !query analysis +Project [time_from_seconds(43200) AS time_from_seconds(43200)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(52200.5) +-- !query analysis +Project [time_from_seconds(52200.5) AS time_from_seconds(52200.5)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(86399.999999) +-- !query analysis +Project [time_from_seconds(86399.999999) AS time_from_seconds(86399.999999)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(-1) +-- !query analysis +Project [time_from_seconds(-1) AS time_from_seconds(-1)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(86400) +-- !query analysis +Project [time_from_seconds(86400) AS time_from_seconds(86400)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(90000) +-- !query analysis +Project [time_from_seconds(90000) AS time_from_seconds(90000)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(NULL) +-- !query analysis +Project [time_from_seconds(null) AS time_from_seconds(NULL)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(0) +-- !query analysis +Project [time_from_millis(0) AS time_from_millis(0)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(43200) +-- !query analysis +Project [time_from_millis(43200) AS time_from_millis(43200)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(52200000) +-- !query analysis +Project [time_from_millis(52200000) AS time_from_millis(52200000)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(52200500) +-- !query analysis +Project [time_from_millis(52200500) AS time_from_millis(52200500)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(86399999) +-- !query analysis +Project [time_from_millis(86399999) AS time_from_millis(86399999)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(-1) +-- !query analysis +Project [time_from_millis(-1) AS time_from_millis(-1)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(86400000) +-- !query analysis +Project [time_from_millis(86400000) AS time_from_millis(86400000)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(NULL) +-- !query analysis +Project [time_from_millis(null) AS time_from_millis(NULL)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(0) +-- !query analysis +Project [time_from_micros(0) AS time_from_micros(0)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(43200) +-- !query analysis +Project [time_from_micros(43200) AS time_from_micros(43200)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(52200000000) +-- !query analysis +Project [time_from_micros(52200000000) AS time_from_micros(52200000000)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(52200500000) +-- !query analysis +Project [time_from_micros(52200500000) AS time_from_micros(52200500000)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(86399999999) +-- !query analysis +Project [time_from_micros(86399999999) AS time_from_micros(86399999999)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(-1) +-- !query analysis +Project [time_from_micros(-1) AS time_from_micros(-1)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(86400000000) +-- !query analysis +Project [time_from_micros(86400000000) AS time_from_micros(86400000000)#x] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(NULL) +-- !query analysis +Project [time_from_micros(null) AS time_from_micros(NULL)#x] ++- OneRowRelation + + +-- !query +SELECT time_to_seconds(TIME'00:00:00') +-- !query analysis +Project [time_to_seconds(00:00:00) AS time_to_seconds(TIME '00:00:00')#x] ++- OneRowRelation + + +-- !query +SELECT time_to_seconds(TIME'12:00:00') +-- !query analysis +Project [time_to_seconds(12:00:00) AS time_to_seconds(TIME '12:00:00')#x] ++- OneRowRelation + + +-- !query +SELECT time_to_seconds(TIME'14:30:00.5') +-- !query analysis +Project [time_to_seconds(14:30:00.5) AS time_to_seconds(TIME '14:30:00.5')#x] ++- OneRowRelation + + +-- !query +SELECT time_to_seconds(TIME'23:59:59.999') +-- !query analysis +Project [time_to_seconds(23:59:59.999) AS time_to_seconds(TIME '23:59:59.999')#x] ++- OneRowRelation + + +-- !query +SELECT time_to_seconds(TIME'23:59:59.999999') +-- !query analysis +Project [time_to_seconds(23:59:59.999999) AS time_to_seconds(TIME '23:59:59.999999')#x] ++- OneRowRelation + + +-- !query +SELECT time_to_seconds(NULL) +-- !query analysis +Project [time_to_seconds(cast(null as time(6))) AS time_to_seconds(NULL)#x] ++- OneRowRelation + + +-- !query +SELECT time_to_millis(TIME'00:00:00') +-- !query analysis +Project [time_to_millis(00:00:00) AS time_to_millis(TIME '00:00:00')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_millis(TIME'14:30:00') +-- !query analysis +Project [time_to_millis(14:30:00) AS time_to_millis(TIME '14:30:00')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_millis(TIME'14:30:00.5') +-- !query analysis +Project [time_to_millis(14:30:00.5) AS time_to_millis(TIME '14:30:00.5')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_millis(TIME'23:59:59.999') +-- !query analysis +Project [time_to_millis(23:59:59.999) AS time_to_millis(TIME '23:59:59.999')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_millis(TIME'23:59:59.999999') +-- !query analysis +Project [time_to_millis(23:59:59.999999) AS time_to_millis(TIME '23:59:59.999999')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_millis(NULL) +-- !query analysis +Project [time_to_millis(null) AS time_to_millis(NULL)#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_micros(TIME'00:00:00') +-- !query analysis +Project [time_to_micros(00:00:00) AS time_to_micros(TIME '00:00:00')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_micros(TIME'14:30:00') +-- !query analysis +Project [time_to_micros(14:30:00) AS time_to_micros(TIME '14:30:00')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_micros(TIME'14:30:00.5') +-- !query analysis +Project [time_to_micros(14:30:00.5) AS time_to_micros(TIME '14:30:00.5')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_micros(TIME'23:59:59.999') +-- !query analysis +Project [time_to_micros(23:59:59.999) AS time_to_micros(TIME '23:59:59.999')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_micros(TIME'23:59:59.999999') +-- !query analysis +Project [time_to_micros(23:59:59.999999) AS time_to_micros(TIME '23:59:59.999999')#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_micros(NULL) +-- !query analysis +Project [time_to_micros(null) AS time_to_micros(NULL)#xL] ++- OneRowRelation + + +-- !query +SELECT time_to_seconds(time_from_seconds(52200.5)) +-- !query analysis +Project [time_to_seconds(time_from_seconds(52200.5)) AS time_to_seconds(time_from_seconds(52200.5))#x] ++- OneRowRelation + + +-- !query +SELECT time_from_seconds(time_to_seconds(TIME'14:30:00.5')) +-- !query analysis +Project [time_from_seconds(time_to_seconds(14:30:00.5)) AS time_from_seconds(time_to_seconds(TIME '14:30:00.5'))#x] ++- OneRowRelation + + +-- !query +SELECT time_to_millis(time_from_millis(52200500)) +-- !query analysis +Project [time_to_millis(time_from_millis(52200500)) AS time_to_millis(time_from_millis(52200500))#xL] ++- OneRowRelation + + +-- !query +SELECT time_from_millis(time_to_millis(TIME'14:30:00.5')) +-- !query analysis +Project [time_from_millis(time_to_millis(14:30:00.5)) AS time_from_millis(time_to_millis(TIME '14:30:00.5'))#x] ++- OneRowRelation + + +-- !query +SELECT time_to_micros(time_from_micros(52200500000)) +-- !query analysis +Project [time_to_micros(time_from_micros(52200500000)) AS time_to_micros(time_from_micros(52200500000))#xL] ++- OneRowRelation + + +-- !query +SELECT time_from_micros(time_to_micros(TIME'14:30:00.5')) +-- !query analysis +Project [time_from_micros(time_to_micros(14:30:00.5)) AS time_from_micros(time_to_micros(TIME '14:30:00.5'))#x] ++- OneRowRelation + + +-- !query +SET spark.sql.ansi.enabled = true +-- !query analysis +SetCommand (spark.sql.ansi.enabled,Some(true)) + + -- !query SELECT time_from_seconds(0) -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/time.sql b/sql/core/src/test/resources/sql-tests/inputs/time.sql index 3e1b62f84cbb..1bb1e02c4576 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/time.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/time.sql @@ -302,38 +302,105 @@ SELECT '12:30:41.123' - TIME'10:00:01'; SELECT '23:59:59.999999' :: TIME(6) - '00:00' :: TIME(0); SELECT '00:00:00.1234' :: TIME(4) - TIME'23:59:59'; --- Numeric constructor functions for TIME type --- time_from_seconds +-- Numeric constructor and extractor functions for TIME type +-- Test with ANSI mode disabled (invalid inputs return NULL) +SET spark.sql.ansi.enabled = false; + +-- time_from_seconds (valid: 0 to 86399.999999) +SELECT time_from_seconds(0); +SELECT time_from_seconds(43200); +SELECT time_from_seconds(52200.5); +SELECT time_from_seconds(86399.999999); +SELECT time_from_seconds(-1); -- invalid: negative +SELECT time_from_seconds(86400); -- invalid: >= 86400 +SELECT time_from_seconds(90000); -- invalid: >= 86400 +SELECT time_from_seconds(NULL); + +-- time_from_millis (valid: 0 to 86399999) +SELECT time_from_millis(0); +SELECT time_from_millis(43200); +SELECT time_from_millis(52200000); +SELECT time_from_millis(52200500); +SELECT time_from_millis(86399999); +SELECT time_from_millis(-1); -- invalid: negative +SELECT time_from_millis(86400000); -- invalid: >= 86400000 +SELECT time_from_millis(NULL); + +-- time_from_micros (valid: 0 to 86399999999) +SELECT time_from_micros(0); +SELECT time_from_micros(43200); +SELECT time_from_micros(52200000000); +SELECT time_from_micros(52200500000); +SELECT time_from_micros(86399999999); +SELECT time_from_micros(-1); -- invalid: negative +SELECT time_from_micros(86400000000); -- invalid: >= 86400000000 +SELECT time_from_micros(NULL); + +-- time_to_seconds +SELECT time_to_seconds(TIME'00:00:00'); +SELECT time_to_seconds(TIME'12:00:00'); +SELECT time_to_seconds(TIME'14:30:00.5'); +SELECT time_to_seconds(TIME'23:59:59.999'); +SELECT time_to_seconds(TIME'23:59:59.999999'); +SELECT time_to_seconds(NULL); + +-- time_to_millis +SELECT time_to_millis(TIME'00:00:00'); +SELECT time_to_millis(TIME'14:30:00'); +SELECT time_to_millis(TIME'14:30:00.5'); +SELECT time_to_millis(TIME'23:59:59.999'); +SELECT time_to_millis(TIME'23:59:59.999999'); +SELECT time_to_millis(NULL); + +-- time_to_micros +SELECT time_to_micros(TIME'00:00:00'); +SELECT time_to_micros(TIME'14:30:00'); +SELECT time_to_micros(TIME'14:30:00.5'); +SELECT time_to_micros(TIME'23:59:59.999'); +SELECT time_to_micros(TIME'23:59:59.999999'); +SELECT time_to_micros(NULL); + +-- Round trip tests +SELECT time_to_seconds(time_from_seconds(52200.5)); +SELECT time_from_seconds(time_to_seconds(TIME'14:30:00.5')); +SELECT time_to_millis(time_from_millis(52200500)); +SELECT time_from_millis(time_to_millis(TIME'14:30:00.5')); +SELECT time_to_micros(time_from_micros(52200500000)); +SELECT time_from_micros(time_to_micros(TIME'14:30:00.5')); + +-- Test with ANSI mode enabled (invalid inputs throw exceptions) +SET spark.sql.ansi.enabled = true; + +-- time_from_seconds (valid: 0 to 86399.999999) SELECT time_from_seconds(0); SELECT time_from_seconds(43200); SELECT time_from_seconds(52200.5); SELECT time_from_seconds(86399.999999); -SELECT time_from_seconds(-1); -SELECT time_from_seconds(86400); -SELECT time_from_seconds(90000); +SELECT time_from_seconds(-1); -- invalid: negative → exception +SELECT time_from_seconds(86400); -- invalid: >= 86400 → exception +SELECT time_from_seconds(90000); -- invalid: >= 86400 → exception SELECT time_from_seconds(NULL); --- time_from_millis +-- time_from_millis (valid: 0 to 86399999) SELECT time_from_millis(0); SELECT time_from_millis(43200); SELECT time_from_millis(52200000); SELECT time_from_millis(52200500); SELECT time_from_millis(86399999); -SELECT time_from_millis(-1); -SELECT time_from_millis(86400000); +SELECT time_from_millis(-1); -- invalid: negative → exception +SELECT time_from_millis(86400000); -- invalid: >= 86400000 → exception SELECT time_from_millis(NULL); --- time_from_micros +-- time_from_micros (valid: 0 to 86399999999) SELECT time_from_micros(0); SELECT time_from_micros(43200); SELECT time_from_micros(52200000000); SELECT time_from_micros(52200500000); SELECT time_from_micros(86399999999); -SELECT time_from_micros(-1); -SELECT time_from_micros(86400000000); +SELECT time_from_micros(-1); -- invalid: negative → exception +SELECT time_from_micros(86400000000); -- invalid: >= 86400000000 → exception SELECT time_from_micros(NULL); --- Numeric extractor functions for TIME type -- time_to_seconds SELECT time_to_seconds(TIME'00:00:00'); SELECT time_to_seconds(TIME'12:00:00'); @@ -358,7 +425,7 @@ SELECT time_to_micros(TIME'23:59:59.999'); SELECT time_to_micros(TIME'23:59:59.999999'); SELECT time_to_micros(NULL); --- Round trip tests for numeric functions +-- Round trip tests SELECT time_to_seconds(time_from_seconds(52200.5)); SELECT time_from_seconds(time_to_seconds(TIME'14:30:00.5')); SELECT time_to_millis(time_from_millis(52200500)); diff --git a/sql/core/src/test/resources/sql-tests/results/time.sql.out b/sql/core/src/test/resources/sql-tests/results/time.sql.out index 802c9a358b55..ab89a595b196 100644 --- a/sql/core/src/test/resources/sql-tests/results/time.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/time.sql.out @@ -2379,6 +2379,14 @@ struct -0 23:59:58.876600000 +-- !query +SET spark.sql.ansi.enabled = false +-- !query schema +struct +-- !query output +spark.sql.ansi.enabled false + + -- !query SELECT time_from_seconds(0) -- !query schema @@ -2755,6 +2763,454 @@ struct 52200500000 +-- !query +SELECT time_from_micros(time_to_micros(TIME'14:30:00.5')) +-- !query schema +struct +-- !query output +14:30:00.5 + + +-- !query +SET spark.sql.ansi.enabled = true +-- !query schema +struct +-- !query output +spark.sql.ansi.enabled true + + +-- !query +SELECT time_from_seconds(0) +-- !query schema +struct +-- !query output +00:00:00 + + +-- !query +SELECT time_from_seconds(43200) +-- !query schema +struct +-- !query output +12:00:00 + + +-- !query +SELECT time_from_seconds(52200.5) +-- !query schema +struct +-- !query output +14:30:00.5 + + +-- !query +SELECT time_from_seconds(86399.999999) +-- !query schema +struct +-- !query output +23:59:59.999999 + + +-- !query +SELECT time_from_seconds(-1) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkDateTimeException +{ + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "sqlState" : "22023", + "messageParameters" : { + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got -1000000000 nanoseconds" + } +} + + +-- !query +SELECT time_from_seconds(86400) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkDateTimeException +{ + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "sqlState" : "22023", + "messageParameters" : { + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 86400000000000 nanoseconds" + } +} + + +-- !query +SELECT time_from_seconds(90000) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkDateTimeException +{ + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "sqlState" : "22023", + "messageParameters" : { + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 90000000000000 nanoseconds" + } +} + + +-- !query +SELECT time_from_seconds(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT time_from_millis(0) +-- !query schema +struct +-- !query output +00:00:00 + + +-- !query +SELECT time_from_millis(43200) +-- !query schema +struct +-- !query output +00:00:43.2 + + +-- !query +SELECT time_from_millis(52200000) +-- !query schema +struct +-- !query output +14:30:00 + + +-- !query +SELECT time_from_millis(52200500) +-- !query schema +struct +-- !query output +14:30:00.5 + + +-- !query +SELECT time_from_millis(86399999) +-- !query schema +struct +-- !query output +23:59:59.999 + + +-- !query +SELECT time_from_millis(-1) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkDateTimeException +{ + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "sqlState" : "22023", + "messageParameters" : { + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got -1000000 nanoseconds" + } +} + + +-- !query +SELECT time_from_millis(86400000) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkDateTimeException +{ + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "sqlState" : "22023", + "messageParameters" : { + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 86400000000000 nanoseconds" + } +} + + +-- !query +SELECT time_from_millis(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT time_from_micros(0) +-- !query schema +struct +-- !query output +00:00:00 + + +-- !query +SELECT time_from_micros(43200) +-- !query schema +struct +-- !query output +00:00:00.0432 + + +-- !query +SELECT time_from_micros(52200000000) +-- !query schema +struct +-- !query output +14:30:00 + + +-- !query +SELECT time_from_micros(52200500000) +-- !query schema +struct +-- !query output +14:30:00.5 + + +-- !query +SELECT time_from_micros(86399999999) +-- !query schema +struct +-- !query output +23:59:59.999999 + + +-- !query +SELECT time_from_micros(-1) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkDateTimeException +{ + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "sqlState" : "22023", + "messageParameters" : { + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got -1000 nanoseconds" + } +} + + +-- !query +SELECT time_from_micros(86400000000) +-- !query schema +struct<> +-- !query output +org.apache.spark.SparkDateTimeException +{ + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "sqlState" : "22023", + "messageParameters" : { + "ansiConfig" : "\"spark.sql.ansi.enabled\"", + "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 86400000000000 nanoseconds" + } +} + + +-- !query +SELECT time_from_micros(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT time_to_seconds(TIME'00:00:00') +-- !query schema +struct +-- !query output +0.000000 + + +-- !query +SELECT time_to_seconds(TIME'12:00:00') +-- !query schema +struct +-- !query output +43200.000000 + + +-- !query +SELECT time_to_seconds(TIME'14:30:00.5') +-- !query schema +struct +-- !query output +52200.500000 + + +-- !query +SELECT time_to_seconds(TIME'23:59:59.999') +-- !query schema +struct +-- !query output +86399.999000 + + +-- !query +SELECT time_to_seconds(TIME'23:59:59.999999') +-- !query schema +struct +-- !query output +86399.999999 + + +-- !query +SELECT time_to_seconds(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT time_to_millis(TIME'00:00:00') +-- !query schema +struct +-- !query output +0 + + +-- !query +SELECT time_to_millis(TIME'14:30:00') +-- !query schema +struct +-- !query output +52200000 + + +-- !query +SELECT time_to_millis(TIME'14:30:00.5') +-- !query schema +struct +-- !query output +52200500 + + +-- !query +SELECT time_to_millis(TIME'23:59:59.999') +-- !query schema +struct +-- !query output +86399999 + + +-- !query +SELECT time_to_millis(TIME'23:59:59.999999') +-- !query schema +struct +-- !query output +86399999 + + +-- !query +SELECT time_to_millis(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT time_to_micros(TIME'00:00:00') +-- !query schema +struct +-- !query output +0 + + +-- !query +SELECT time_to_micros(TIME'14:30:00') +-- !query schema +struct +-- !query output +52200000000 + + +-- !query +SELECT time_to_micros(TIME'14:30:00.5') +-- !query schema +struct +-- !query output +52200500000 + + +-- !query +SELECT time_to_micros(TIME'23:59:59.999') +-- !query schema +struct +-- !query output +86399999000 + + +-- !query +SELECT time_to_micros(TIME'23:59:59.999999') +-- !query schema +struct +-- !query output +86399999999 + + +-- !query +SELECT time_to_micros(NULL) +-- !query schema +struct +-- !query output +NULL + + +-- !query +SELECT time_to_seconds(time_from_seconds(52200.5)) +-- !query schema +struct +-- !query output +52200.500000 + + +-- !query +SELECT time_from_seconds(time_to_seconds(TIME'14:30:00.5')) +-- !query schema +struct +-- !query output +14:30:00.5 + + +-- !query +SELECT time_to_millis(time_from_millis(52200500)) +-- !query schema +struct +-- !query output +52200500 + + +-- !query +SELECT time_from_millis(time_to_millis(TIME'14:30:00.5')) +-- !query schema +struct +-- !query output +14:30:00.5 + + +-- !query +SELECT time_to_micros(time_from_micros(52200500000)) +-- !query schema +struct +-- !query output +52200500000 + + -- !query SELECT time_from_micros(time_to_micros(TIME'14:30:00.5')) -- !query schema From 3b8ca130e7bb515d30224ffc2ecd98b211284e62 Mon Sep 17 00:00:00 2001 From: vinodkc Date: Mon, 8 Dec 2025 19:27:17 -0800 Subject: [PATCH 3/5] Fix document generation issue --- .../spark/sql/catalyst/expressions/timeExpressions.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 89e546701783..092539aa394f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -872,10 +872,6 @@ abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes { 14:30:00.5 > SELECT _FUNC_(86399.999999); 23:59:59.999999 - > SELECT _FUNC_(90000); - NULL - > SELECT _FUNC_(-1); - NULL """, since = "4.2.0", group = "datetime_funcs") From 21304987c1514d7b6c5c5a8795acb52fa67703f2 Mon Sep 17 00:00:00 2001 From: vinodkc Date: Tue, 9 Dec 2025 14:33:49 -0800 Subject: [PATCH 4/5] Fixed review comments --- .../expressions/timeExpressions.scala | 224 +++------- .../sql/catalyst/util/DateTimeUtils.scala | 105 ++--- .../expressions/TimeExpressionsSuite.scala | 112 ++--- .../sql-tests/analyzer-results/time.sql.out | 348 --------------- .../test/resources/sql-tests/inputs/time.sql | 67 --- .../resources/sql-tests/results/time.sql.out | 400 ------------------ 6 files changed, 157 insertions(+), 1099 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 092539aa394f..72bb55360de5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -25,92 +25,18 @@ import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.analysis.{ExpressionBuilder, TypeCheckResult} import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, TypeCheckSuccess} import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLExpr, toSQLId, toSQLType, toSQLValue} -import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} import org.apache.spark.sql.catalyst.trees.TreePattern.{CURRENT_LIKE, TreePattern} -import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.catalyst.util.TimeFormatter import org.apache.spark.sql.catalyst.util.TypeUtils.ordinalNumber import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.types.StringTypeWithCollation import org.apache.spark.sql.types.{AbstractDataType, AnyTimeType, ByteType, DataType, DayTimeIntervalType, DecimalType, IntegerType, IntegralType, LongType, NumericType, ObjectType, TimeType} import org.apache.spark.sql.types.DayTimeIntervalType.{HOUR, SECOND} import org.apache.spark.unsafe.types.UTF8String -/** - * Helper trait for TIME conversion expressions with consistent error handling. - */ -trait TimeConversionErrorHandling { - def failOnError: Boolean - - /** Wraps evaluation with error handling (throws in ANSI mode, null otherwise). */ - protected def evalWithErrorHandling[T](f: => T): Any = { - try { - f - } catch { - case e: DateTimeException if failOnError => - throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e) - case e: ArithmeticException if failOnError => - throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange( - new DateTimeException(s"Overflow in TIME conversion: ${e.getMessage}")) - case _: DateTimeException | _: ArithmeticException => null - } - } - - /** Generates error handling code (DateTimeException + ArithmeticException). */ - protected def doGenErrorHandling( - ctx: CodegenContext, - ev: ExprCode, - utilCall: String): String = { - val dateTimeErrorBranch = if (failOnError) { - "throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e);" - } else { - s"${ev.isNull} = true;" - } - - val arithmeticErrorBranch = if (failOnError) { - s""" - |throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange( - | new java.time.DateTimeException("Overflow in TIME conversion: " + e.getMessage())); - |""".stripMargin - } else { - s"${ev.isNull} = true;" - } - - s""" - |try { - | ${ev.value} = $utilCall; - |} catch (java.time.DateTimeException e) { - | $dateTimeErrorBranch - |} catch (java.lang.ArithmeticException e) { - | $arithmeticErrorBranch - |} - |""".stripMargin - } - - /** Generates error handling code (DateTimeException only). */ - protected def doGenDateTimeError( - ctx: CodegenContext, - ev: ExprCode, - utilCall: String): String = { - val errorBranch = if (failOnError) { - "throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e);" - } else { - s"${ev.isNull} = true;" - } - s""" - |try { - | ${ev.value} = $utilCall; - |} catch (java.time.DateTimeException e) { - | $errorBranch - |} - |""".stripMargin - } -} - /** * Parses a column to a time based on the given format. */ @@ -824,36 +750,6 @@ case class TimeTrunc(unit: Expression, time: Expression) } } -abstract class IntegralToTimeBase - extends UnaryExpression with ExpectsInputTypes with TimeConversionErrorHandling { - protected def upScaleFactor: Long - def failOnError: Boolean = SQLConf.get.ansiEnabled - - override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) - override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) - override def nullable: Boolean = true - override def nullIntolerant: Boolean = true - - override protected def nullSafeEval(input: Any): Any = { - evalWithErrorHandling { - DateTimeUtils.timeFromIntegral(input.asInstanceOf[Number].longValue(), upScaleFactor) - } - } - - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") - nullSafeCodeGen(ctx, ev, c => - doGenErrorHandling(ctx, ev, s"$dtu.timeFromIntegral($c, ${upScaleFactor}L)") - ) - } -} - -abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes { - override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) - override def dataType: DataType = LongType - override def nullIntolerant: Boolean = true -} - // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(seconds) - Creates a TIME value from seconds since midnight.", @@ -877,28 +773,18 @@ abstract class TimeToLongBase extends UnaryExpression with ExpectsInputTypes { group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeFromSeconds(child: Expression) - extends UnaryExpression with ExpectsInputTypes with TimeConversionErrorHandling { - override def inputTypes: Seq[AbstractDataType] = Seq(NumericType) - override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) - override def nullable: Boolean = true - override def nullIntolerant: Boolean = true - - def failOnError: Boolean = SQLConf.get.ansiEnabled + extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - override def nullSafeEval(input: Any): Any = { - evalWithErrorHandling { - DateTimeUtils.timeFromSeconds(input, child.dataType) - } - } - - override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") - val dt = ctx.addReferenceObj("childDataType", child.dataType) - nullSafeCodeGen(ctx, ev, c => - doGenErrorHandling(ctx, ev, s"$dtu.timeFromSeconds($c, $dt)") - ) - } + override def replacement: Expression = StaticInvoke( + classOf[DateTimeUtils.type], + TimeType(TimeType.MICROS_PRECISION), + "timeFromSeconds", + Seq(child), + Seq(child.dataType) + ) + override def inputTypes: Seq[AbstractDataType] = Seq(NumericType) + override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) override def prettyName: String = "time_from_seconds" override protected def withNewChildInternal(newChild: Expression): TimeFromSeconds = @@ -927,10 +813,18 @@ case class TimeFromSeconds(child: Expression) group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeFromMillis(child: Expression) - extends IntegralToTimeBase { + extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - override def upScaleFactor: Long = NANOS_PER_MILLIS + override def replacement: Expression = StaticInvoke( + classOf[DateTimeUtils.type], + TimeType(TimeType.MICROS_PRECISION), + "timeFromMillis", + Seq(child), + Seq(child.dataType) + ) + override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) + override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) override def prettyName: String = "time_from_millis" override protected def withNewChildInternal(newChild: Expression): TimeFromMillis = @@ -959,10 +853,18 @@ case class TimeFromMillis(child: Expression) group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeFromMicros(child: Expression) - extends IntegralToTimeBase { + extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - override def upScaleFactor: Long = NANOS_PER_MICROS + override def replacement: Expression = StaticInvoke( + classOf[DateTimeUtils.type], + TimeType(TimeType.MICROS_PRECISION), + "timeFromMicros", + Seq(child), + Seq(child.dataType) + ) + override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) + override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) override def prettyName: String = "time_from_micros" override protected def withNewChildInternal(newChild: Expression): TimeFromMicros = @@ -992,28 +894,18 @@ case class TimeFromMicros(child: Expression) group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeToSeconds(child: Expression) - extends UnaryExpression with ImplicitCastInputTypes with TimeConversionErrorHandling { + extends UnaryExpression with RuntimeReplaceable with ImplicitCastInputTypes { + + override def replacement: Expression = StaticInvoke( + classOf[DateTimeUtils.type], + DecimalType(14, 6), + "timeToSeconds", + Seq(child), + Seq(child.dataType) + ) override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) override def dataType: DataType = DecimalType(14, 6) - override def nullable: Boolean = true - override def nullIntolerant: Boolean = true - - def failOnError: Boolean = SQLConf.get.ansiEnabled - - protected override def nullSafeEval(input: Any): Any = { - evalWithErrorHandling { - DateTimeUtils.timeToSeconds(input.asInstanceOf[Long]) - } - } - - override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") - nullSafeCodeGen(ctx, ev, nanos => - doGenDateTimeError(ctx, ev, s"$dtu.timeToSeconds($nanos)") - ) - } - override def prettyName: String = "time_to_seconds" override protected def withNewChildInternal(newChild: Expression): TimeToSeconds = @@ -1043,17 +935,18 @@ case class TimeToSeconds(child: Expression) group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeToMillis(child: Expression) - extends TimeToLongBase { - - override def nullSafeEval(input: Any): Any = { - DateTimeUtils.timeToMillis(input.asInstanceOf[Number].longValue()) - } + extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") - defineCodeGen(ctx, ev, c => s"$dtu.timeToMillis($c)") - } + override def replacement: Expression = StaticInvoke( + classOf[DateTimeUtils.type], + LongType, + "timeToMillis", + Seq(child), + Seq(child.dataType) + ) + override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) + override def dataType: DataType = LongType override def prettyName: String = "time_to_millis" override protected def withNewChildInternal(newChild: Expression): TimeToMillis = @@ -1083,17 +976,18 @@ case class TimeToMillis(child: Expression) group = "datetime_funcs") // scalastyle:on line.size.limit case class TimeToMicros(child: Expression) - extends TimeToLongBase { - - override def nullSafeEval(input: Any): Any = { - DateTimeUtils.timeToMicros(input.asInstanceOf[Number].longValue()) - } + extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { - val dtu = DateTimeUtils.getClass.getName.stripSuffix("$") - defineCodeGen(ctx, ev, c => s"$dtu.timeToMicros($c)") - } + override def replacement: Expression = StaticInvoke( + classOf[DateTimeUtils.type], + LongType, + "timeToMicros", + Seq(child), + Seq(child.dataType) + ) + override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) + override def dataType: DataType = LongType override def prettyName: String = "time_to_micros" override protected def withNewChildInternal(newChild: Expression): TimeToMicros = diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 2017ad9de792..6739d640e68e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -28,7 +28,7 @@ import scala.util.control.NonFatal import org.apache.spark.{QueryContext, SparkException, SparkIllegalArgumentException} import org.apache.spark.sql.catalyst.util.DateTimeConstants._ import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.types.{DataType, Decimal, DecimalType, DoubleExactNumeric, DoubleType, FloatType, IntegralType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{Decimal, DoubleExactNumeric, TimestampNTZType, TimestampType} import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String} /** @@ -897,76 +897,83 @@ object DateTimeUtils extends SparkDateTimeUtils { } } + private def withTimeConversionErrorHandling(f: => Long): Long = { + try { + val nanos = f + if (nanos < 0 || nanos >= NANOS_PER_DAY) { + throw new DateTimeException( + s"Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, " + + s"but got $nanos nanoseconds") + } + nanos + } catch { + case e: DateTimeException => + throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e) + case e: ArithmeticException => + val wrapped = new DateTimeException(s"Overflow in TIME conversion: ${e.getMessage}", e) + throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(wrapped) + } + } + /** - * Creates a TIME value from seconds since midnight. - * @param seconds Numeric value (0 to 86399.999999) - * @param dataType Input data type + * Creates a TIME value from seconds since midnight (integral types). + * @param seconds Seconds (0 to 86399) * @return Nanoseconds since midnight */ - def timeFromSeconds(seconds: Any, dataType: DataType): Long = { - val nanos = dataType match { - case _: IntegralType => - Math.multiplyExact(seconds.asInstanceOf[Number].longValue(), NANOS_PER_SECOND) - case _: DecimalType => - val operand = new java.math.BigDecimal(NANOS_PER_SECOND) - seconds.asInstanceOf[Decimal].toJavaBigDecimal.multiply(operand).longValueExact() - case FloatType => - val f = seconds.asInstanceOf[Float] - if (f.isNaN || f.isInfinite) { - throw new DateTimeException("Cannot convert NaN or Infinite value to TIME") - } - (f.toDouble * NANOS_PER_SECOND).toLong - case DoubleType => - val d = seconds.asInstanceOf[Double] - if (d.isNaN || d.isInfinite) { - throw new DateTimeException("Cannot convert NaN or Infinite value to TIME") - } - (d * NANOS_PER_SECOND).toLong - } - validateTimeNanos(nanos) + def timeFromSeconds(seconds: Long): Long = withTimeConversionErrorHandling { + Math.multiplyExact(seconds, NANOS_PER_SECOND) } /** - * Creates a TIME value from milliseconds since midnight. - * @param millis Milliseconds (0 to 86399999) + * Creates a TIME value from seconds since midnight (decimal type). + * @param seconds Seconds (0 to 86399.999999) * @return Nanoseconds since midnight */ - def timeFromMillis(millis: Long): Long = { - timeFromIntegral(millis, NANOS_PER_MILLIS) + def timeFromSeconds(seconds: Decimal): Long = withTimeConversionErrorHandling { + val operand = new java.math.BigDecimal(NANOS_PER_SECOND) + seconds.toJavaBigDecimal.multiply(operand).longValueExact() } /** - * Creates a TIME value from microseconds since midnight. - * @param micros Microseconds (0 to 86399999999) + * Creates a TIME value from seconds since midnight (float type). + * @param seconds Seconds (0 to 86399.999999) + * @return Nanoseconds since midnight + */ + def timeFromSeconds(seconds: Float): Long = withTimeConversionErrorHandling { + if (seconds.isNaN || seconds.isInfinite) { + throw new DateTimeException("Cannot convert NaN or Infinite value to TIME") + } + (seconds.toDouble * NANOS_PER_SECOND).toLong + } + + /** + * Creates a TIME value from seconds since midnight (double type). + * @param seconds Seconds (0 to 86399.999999) * @return Nanoseconds since midnight */ - def timeFromMicros(micros: Long): Long = { - timeFromIntegral(micros, NANOS_PER_MICROS) + def timeFromSeconds(seconds: Double): Long = withTimeConversionErrorHandling { + if (seconds.isNaN || seconds.isInfinite) { + throw new DateTimeException("Cannot convert NaN or Infinite value to TIME") + } + (seconds * NANOS_PER_SECOND).toLong } /** - * Creates a TIME value from an integral value with a scale factor. - * @param value Integral value - * @param scaleFactor Conversion factor to nanoseconds + * Creates a TIME value from milliseconds since midnight. + * @param millis Milliseconds (0 to 86399999) * @return Nanoseconds since midnight */ - def timeFromIntegral(value: Long, scaleFactor: Long): Long = { - val nanos = if (scaleFactor == 1) value else Math.multiplyExact(value, scaleFactor) - validateTimeNanos(nanos) + def timeFromMillis(millis: Long): Long = withTimeConversionErrorHandling { + Math.multiplyExact(millis, NANOS_PER_MILLIS) } /** - * Validates nanoseconds is within valid TIME range [0, NANOS_PER_DAY). - * @param nanos Nanoseconds to validate - * @return Input nanos if valid + * Creates a TIME value from microseconds since midnight. + * @param micros Microseconds (0 to 86399999999) + * @return Nanoseconds since midnight */ - private def validateTimeNanos(nanos: Long): Long = { - if (nanos < 0 || nanos >= NANOS_PER_DAY) { - throw new DateTimeException( - s"Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, " + - s"but got $nanos nanoseconds") - } - nanos + def timeFromMicros(micros: Long): Long = withTimeConversionErrorHandling { + Math.multiplyExact(micros, NANOS_PER_MICROS) } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala index a1e92e959432..a806e2c9419c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/TimeExpressionsSuite.scala @@ -25,7 +25,6 @@ import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLId, toSQLValue} import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._ import org.apache.spark.sql.catalyst.util.SparkDateTimeUtils.localTimeToNanos -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal, DecimalType, IntegerType, LongType, StringType, TimeType} import org.apache.spark.sql.types.DayTimeIntervalType.{DAY, HOUR, SECOND} @@ -594,76 +593,49 @@ class TimeExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } test("Numeric to TIME conversions - range validation") { - // Test non-ANSI mode: returns NULL for out-of-range values - withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { - // time_from_seconds - out of range [0, 86400) - checkEvaluation(TimeFromSeconds(Literal(-1L)), null) - checkEvaluation(TimeFromSeconds(Literal(86400L)), null) - checkEvaluation(TimeFromSeconds(Literal(90000L)), null) - checkEvaluation(TimeFromSeconds(Literal(Decimal(-0.1))), null) - checkEvaluation(TimeFromSeconds(Literal(Decimal(86400.0))), null) - - // time_from_millis - out of range [0, 86400000) - checkEvaluation(TimeFromMillis(Literal(-1L)), null) - checkEvaluation(TimeFromMillis(Literal(86400000L)), null) - - // time_from_micros - out of range [0, 86400000000) - checkEvaluation(TimeFromMicros(Literal(-1L)), null) - checkEvaluation(TimeFromMicros(Literal(86400000000L)), null) - - // Test overflow in TIME conversion - returns NULL in non-ANSI mode - checkEvaluation(TimeFromSeconds(Literal(Long.MaxValue)), null) - - // Test NaN and Infinite for floating point - returns NULL in non-ANSI mode - checkEvaluation(TimeFromSeconds(Literal(Float.NaN)), null) - checkEvaluation(TimeFromSeconds(Literal(Double.PositiveInfinity)), null) - } - // Test ANSI mode: throws exceptions for out-of-range values - withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") { - // time_from_seconds - out of range [0, 86400) - checkExceptionInExpression[SparkDateTimeException]( - TimeFromSeconds(Literal(-1L)), - "Invalid TIME value") - checkExceptionInExpression[SparkDateTimeException]( - TimeFromSeconds(Literal(86400L)), - "Invalid TIME value") - checkExceptionInExpression[SparkDateTimeException]( - TimeFromSeconds(Literal(Decimal(-0.1))), - "Invalid TIME value") - checkExceptionInExpression[SparkDateTimeException]( - TimeFromSeconds(Literal(Decimal(86400.0))), - "Invalid TIME value") - - // time_from_millis - out of range [0, 86400000) - checkExceptionInExpression[SparkDateTimeException]( - TimeFromMillis(Literal(-1L)), - "Invalid TIME value") - checkExceptionInExpression[SparkDateTimeException]( - TimeFromMillis(Literal(86400000L)), - "Invalid TIME value") - - // time_from_micros - out of range [0, 86400000000) - checkExceptionInExpression[SparkDateTimeException]( - TimeFromMicros(Literal(-1L)), - "Invalid TIME value") - checkExceptionInExpression[SparkDateTimeException]( - TimeFromMicros(Literal(86400000000L)), - "Invalid TIME value") - - // Test overflow in TIME conversion - checkExceptionInExpression[SparkDateTimeException]( - TimeFromSeconds(Literal(Long.MaxValue)), - "Overflow in TIME conversion") - - // Test NaN and Infinite for floating point - checkExceptionInExpression[SparkDateTimeException]( - TimeFromSeconds(Literal(Float.NaN)), - "Cannot convert NaN or Infinite value to TIME") - checkExceptionInExpression[SparkDateTimeException]( - TimeFromSeconds(Literal(Double.PositiveInfinity)), - "Cannot convert NaN or Infinite value to TIME") - } + // time_from_seconds - out of range [0, 86400) + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(-1L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(86400L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Decimal(-0.1))), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Decimal(86400.0))), + "Invalid TIME value") + + // time_from_millis - out of range [0, 86400000) + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMillis(Literal(-1L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMillis(Literal(86400000L)), + "Invalid TIME value") + + // time_from_micros - out of range [0, 86400000000) + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMicros(Literal(-1L)), + "Invalid TIME value") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromMicros(Literal(86400000000L)), + "Invalid TIME value") + + // Test overflow in TIME conversion + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Long.MaxValue)), + "Overflow in TIME conversion") + + // Test NaN and Infinite for floating point + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Float.NaN)), + "Cannot convert NaN or Infinite value to TIME") + checkExceptionInExpression[SparkDateTimeException]( + TimeFromSeconds(Literal(Double.PositiveInfinity)), + "Cannot convert NaN or Infinite value to TIME") } test("Numeric to TIME conversions - NULL inputs") { diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out index bcf0c4cb6da5..7075a9f8c4b4 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out @@ -1960,354 +1960,6 @@ Project [cast(00:00:00.1234 as time(4)) - 23:59:59 AS CAST(00:00:00.1234 AS TIME +- OneRowRelation --- !query -SET spark.sql.ansi.enabled = false --- !query analysis -SetCommand (spark.sql.ansi.enabled,Some(false)) - - --- !query -SELECT time_from_seconds(0) --- !query analysis -Project [time_from_seconds(0) AS time_from_seconds(0)#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(43200) --- !query analysis -Project [time_from_seconds(43200) AS time_from_seconds(43200)#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(52200.5) --- !query analysis -Project [time_from_seconds(52200.5) AS time_from_seconds(52200.5)#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(86399.999999) --- !query analysis -Project [time_from_seconds(86399.999999) AS time_from_seconds(86399.999999)#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(-1) --- !query analysis -Project [time_from_seconds(-1) AS time_from_seconds(-1)#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(86400) --- !query analysis -Project [time_from_seconds(86400) AS time_from_seconds(86400)#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(90000) --- !query analysis -Project [time_from_seconds(90000) AS time_from_seconds(90000)#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(NULL) --- !query analysis -Project [time_from_seconds(null) AS time_from_seconds(NULL)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(0) --- !query analysis -Project [time_from_millis(0) AS time_from_millis(0)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(43200) --- !query analysis -Project [time_from_millis(43200) AS time_from_millis(43200)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(52200000) --- !query analysis -Project [time_from_millis(52200000) AS time_from_millis(52200000)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(52200500) --- !query analysis -Project [time_from_millis(52200500) AS time_from_millis(52200500)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(86399999) --- !query analysis -Project [time_from_millis(86399999) AS time_from_millis(86399999)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(-1) --- !query analysis -Project [time_from_millis(-1) AS time_from_millis(-1)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(86400000) --- !query analysis -Project [time_from_millis(86400000) AS time_from_millis(86400000)#x] -+- OneRowRelation - - --- !query -SELECT time_from_millis(NULL) --- !query analysis -Project [time_from_millis(null) AS time_from_millis(NULL)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(0) --- !query analysis -Project [time_from_micros(0) AS time_from_micros(0)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(43200) --- !query analysis -Project [time_from_micros(43200) AS time_from_micros(43200)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(52200000000) --- !query analysis -Project [time_from_micros(52200000000) AS time_from_micros(52200000000)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(52200500000) --- !query analysis -Project [time_from_micros(52200500000) AS time_from_micros(52200500000)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(86399999999) --- !query analysis -Project [time_from_micros(86399999999) AS time_from_micros(86399999999)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(-1) --- !query analysis -Project [time_from_micros(-1) AS time_from_micros(-1)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(86400000000) --- !query analysis -Project [time_from_micros(86400000000) AS time_from_micros(86400000000)#x] -+- OneRowRelation - - --- !query -SELECT time_from_micros(NULL) --- !query analysis -Project [time_from_micros(null) AS time_from_micros(NULL)#x] -+- OneRowRelation - - --- !query -SELECT time_to_seconds(TIME'00:00:00') --- !query analysis -Project [time_to_seconds(00:00:00) AS time_to_seconds(TIME '00:00:00')#x] -+- OneRowRelation - - --- !query -SELECT time_to_seconds(TIME'12:00:00') --- !query analysis -Project [time_to_seconds(12:00:00) AS time_to_seconds(TIME '12:00:00')#x] -+- OneRowRelation - - --- !query -SELECT time_to_seconds(TIME'14:30:00.5') --- !query analysis -Project [time_to_seconds(14:30:00.5) AS time_to_seconds(TIME '14:30:00.5')#x] -+- OneRowRelation - - --- !query -SELECT time_to_seconds(TIME'23:59:59.999') --- !query analysis -Project [time_to_seconds(23:59:59.999) AS time_to_seconds(TIME '23:59:59.999')#x] -+- OneRowRelation - - --- !query -SELECT time_to_seconds(TIME'23:59:59.999999') --- !query analysis -Project [time_to_seconds(23:59:59.999999) AS time_to_seconds(TIME '23:59:59.999999')#x] -+- OneRowRelation - - --- !query -SELECT time_to_seconds(NULL) --- !query analysis -Project [time_to_seconds(cast(null as time(6))) AS time_to_seconds(NULL)#x] -+- OneRowRelation - - --- !query -SELECT time_to_millis(TIME'00:00:00') --- !query analysis -Project [time_to_millis(00:00:00) AS time_to_millis(TIME '00:00:00')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_millis(TIME'14:30:00') --- !query analysis -Project [time_to_millis(14:30:00) AS time_to_millis(TIME '14:30:00')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_millis(TIME'14:30:00.5') --- !query analysis -Project [time_to_millis(14:30:00.5) AS time_to_millis(TIME '14:30:00.5')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_millis(TIME'23:59:59.999') --- !query analysis -Project [time_to_millis(23:59:59.999) AS time_to_millis(TIME '23:59:59.999')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_millis(TIME'23:59:59.999999') --- !query analysis -Project [time_to_millis(23:59:59.999999) AS time_to_millis(TIME '23:59:59.999999')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_millis(NULL) --- !query analysis -Project [time_to_millis(null) AS time_to_millis(NULL)#xL] -+- OneRowRelation - - --- !query -SELECT time_to_micros(TIME'00:00:00') --- !query analysis -Project [time_to_micros(00:00:00) AS time_to_micros(TIME '00:00:00')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_micros(TIME'14:30:00') --- !query analysis -Project [time_to_micros(14:30:00) AS time_to_micros(TIME '14:30:00')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_micros(TIME'14:30:00.5') --- !query analysis -Project [time_to_micros(14:30:00.5) AS time_to_micros(TIME '14:30:00.5')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_micros(TIME'23:59:59.999') --- !query analysis -Project [time_to_micros(23:59:59.999) AS time_to_micros(TIME '23:59:59.999')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_micros(TIME'23:59:59.999999') --- !query analysis -Project [time_to_micros(23:59:59.999999) AS time_to_micros(TIME '23:59:59.999999')#xL] -+- OneRowRelation - - --- !query -SELECT time_to_micros(NULL) --- !query analysis -Project [time_to_micros(null) AS time_to_micros(NULL)#xL] -+- OneRowRelation - - --- !query -SELECT time_to_seconds(time_from_seconds(52200.5)) --- !query analysis -Project [time_to_seconds(time_from_seconds(52200.5)) AS time_to_seconds(time_from_seconds(52200.5))#x] -+- OneRowRelation - - --- !query -SELECT time_from_seconds(time_to_seconds(TIME'14:30:00.5')) --- !query analysis -Project [time_from_seconds(time_to_seconds(14:30:00.5)) AS time_from_seconds(time_to_seconds(TIME '14:30:00.5'))#x] -+- OneRowRelation - - --- !query -SELECT time_to_millis(time_from_millis(52200500)) --- !query analysis -Project [time_to_millis(time_from_millis(52200500)) AS time_to_millis(time_from_millis(52200500))#xL] -+- OneRowRelation - - --- !query -SELECT time_from_millis(time_to_millis(TIME'14:30:00.5')) --- !query analysis -Project [time_from_millis(time_to_millis(14:30:00.5)) AS time_from_millis(time_to_millis(TIME '14:30:00.5'))#x] -+- OneRowRelation - - --- !query -SELECT time_to_micros(time_from_micros(52200500000)) --- !query analysis -Project [time_to_micros(time_from_micros(52200500000)) AS time_to_micros(time_from_micros(52200500000))#xL] -+- OneRowRelation - - --- !query -SELECT time_from_micros(time_to_micros(TIME'14:30:00.5')) --- !query analysis -Project [time_from_micros(time_to_micros(14:30:00.5)) AS time_from_micros(time_to_micros(TIME '14:30:00.5'))#x] -+- OneRowRelation - - --- !query -SET spark.sql.ansi.enabled = true --- !query analysis -SetCommand (spark.sql.ansi.enabled,Some(true)) - - -- !query SELECT time_from_seconds(0) -- !query analysis diff --git a/sql/core/src/test/resources/sql-tests/inputs/time.sql b/sql/core/src/test/resources/sql-tests/inputs/time.sql index 1bb1e02c4576..a0e1ba5973be 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/time.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/time.sql @@ -303,73 +303,6 @@ SELECT '23:59:59.999999' :: TIME(6) - '00:00' :: TIME(0); SELECT '00:00:00.1234' :: TIME(4) - TIME'23:59:59'; -- Numeric constructor and extractor functions for TIME type --- Test with ANSI mode disabled (invalid inputs return NULL) -SET spark.sql.ansi.enabled = false; - --- time_from_seconds (valid: 0 to 86399.999999) -SELECT time_from_seconds(0); -SELECT time_from_seconds(43200); -SELECT time_from_seconds(52200.5); -SELECT time_from_seconds(86399.999999); -SELECT time_from_seconds(-1); -- invalid: negative -SELECT time_from_seconds(86400); -- invalid: >= 86400 -SELECT time_from_seconds(90000); -- invalid: >= 86400 -SELECT time_from_seconds(NULL); - --- time_from_millis (valid: 0 to 86399999) -SELECT time_from_millis(0); -SELECT time_from_millis(43200); -SELECT time_from_millis(52200000); -SELECT time_from_millis(52200500); -SELECT time_from_millis(86399999); -SELECT time_from_millis(-1); -- invalid: negative -SELECT time_from_millis(86400000); -- invalid: >= 86400000 -SELECT time_from_millis(NULL); - --- time_from_micros (valid: 0 to 86399999999) -SELECT time_from_micros(0); -SELECT time_from_micros(43200); -SELECT time_from_micros(52200000000); -SELECT time_from_micros(52200500000); -SELECT time_from_micros(86399999999); -SELECT time_from_micros(-1); -- invalid: negative -SELECT time_from_micros(86400000000); -- invalid: >= 86400000000 -SELECT time_from_micros(NULL); - --- time_to_seconds -SELECT time_to_seconds(TIME'00:00:00'); -SELECT time_to_seconds(TIME'12:00:00'); -SELECT time_to_seconds(TIME'14:30:00.5'); -SELECT time_to_seconds(TIME'23:59:59.999'); -SELECT time_to_seconds(TIME'23:59:59.999999'); -SELECT time_to_seconds(NULL); - --- time_to_millis -SELECT time_to_millis(TIME'00:00:00'); -SELECT time_to_millis(TIME'14:30:00'); -SELECT time_to_millis(TIME'14:30:00.5'); -SELECT time_to_millis(TIME'23:59:59.999'); -SELECT time_to_millis(TIME'23:59:59.999999'); -SELECT time_to_millis(NULL); - --- time_to_micros -SELECT time_to_micros(TIME'00:00:00'); -SELECT time_to_micros(TIME'14:30:00'); -SELECT time_to_micros(TIME'14:30:00.5'); -SELECT time_to_micros(TIME'23:59:59.999'); -SELECT time_to_micros(TIME'23:59:59.999999'); -SELECT time_to_micros(NULL); - --- Round trip tests -SELECT time_to_seconds(time_from_seconds(52200.5)); -SELECT time_from_seconds(time_to_seconds(TIME'14:30:00.5')); -SELECT time_to_millis(time_from_millis(52200500)); -SELECT time_from_millis(time_to_millis(TIME'14:30:00.5')); -SELECT time_to_micros(time_from_micros(52200500000)); -SELECT time_from_micros(time_to_micros(TIME'14:30:00.5')); - --- Test with ANSI mode enabled (invalid inputs throw exceptions) -SET spark.sql.ansi.enabled = true; -- time_from_seconds (valid: 0 to 86399.999999) SELECT time_from_seconds(0); diff --git a/sql/core/src/test/resources/sql-tests/results/time.sql.out b/sql/core/src/test/resources/sql-tests/results/time.sql.out index ab89a595b196..033f9be598f2 100644 --- a/sql/core/src/test/resources/sql-tests/results/time.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/time.sql.out @@ -2379,406 +2379,6 @@ struct -0 23:59:58.876600000 --- !query -SET spark.sql.ansi.enabled = false --- !query schema -struct --- !query output -spark.sql.ansi.enabled false - - --- !query -SELECT time_from_seconds(0) --- !query schema -struct --- !query output -00:00:00 - - --- !query -SELECT time_from_seconds(43200) --- !query schema -struct --- !query output -12:00:00 - - --- !query -SELECT time_from_seconds(52200.5) --- !query schema -struct --- !query output -14:30:00.5 - - --- !query -SELECT time_from_seconds(86399.999999) --- !query schema -struct --- !query output -23:59:59.999999 - - --- !query -SELECT time_from_seconds(-1) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_seconds(86400) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_seconds(90000) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_seconds(NULL) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_millis(0) --- !query schema -struct --- !query output -00:00:00 - - --- !query -SELECT time_from_millis(43200) --- !query schema -struct --- !query output -00:00:43.2 - - --- !query -SELECT time_from_millis(52200000) --- !query schema -struct --- !query output -14:30:00 - - --- !query -SELECT time_from_millis(52200500) --- !query schema -struct --- !query output -14:30:00.5 - - --- !query -SELECT time_from_millis(86399999) --- !query schema -struct --- !query output -23:59:59.999 - - --- !query -SELECT time_from_millis(-1) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_millis(86400000) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_millis(NULL) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_micros(0) --- !query schema -struct --- !query output -00:00:00 - - --- !query -SELECT time_from_micros(43200) --- !query schema -struct --- !query output -00:00:00.0432 - - --- !query -SELECT time_from_micros(52200000000) --- !query schema -struct --- !query output -14:30:00 - - --- !query -SELECT time_from_micros(52200500000) --- !query schema -struct --- !query output -14:30:00.5 - - --- !query -SELECT time_from_micros(86399999999) --- !query schema -struct --- !query output -23:59:59.999999 - - --- !query -SELECT time_from_micros(-1) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_micros(86400000000) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_from_micros(NULL) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_to_seconds(TIME'00:00:00') --- !query schema -struct --- !query output -0.000000 - - --- !query -SELECT time_to_seconds(TIME'12:00:00') --- !query schema -struct --- !query output -43200.000000 - - --- !query -SELECT time_to_seconds(TIME'14:30:00.5') --- !query schema -struct --- !query output -52200.500000 - - --- !query -SELECT time_to_seconds(TIME'23:59:59.999') --- !query schema -struct --- !query output -86399.999000 - - --- !query -SELECT time_to_seconds(TIME'23:59:59.999999') --- !query schema -struct --- !query output -86399.999999 - - --- !query -SELECT time_to_seconds(NULL) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_to_millis(TIME'00:00:00') --- !query schema -struct --- !query output -0 - - --- !query -SELECT time_to_millis(TIME'14:30:00') --- !query schema -struct --- !query output -52200000 - - --- !query -SELECT time_to_millis(TIME'14:30:00.5') --- !query schema -struct --- !query output -52200500 - - --- !query -SELECT time_to_millis(TIME'23:59:59.999') --- !query schema -struct --- !query output -86399999 - - --- !query -SELECT time_to_millis(TIME'23:59:59.999999') --- !query schema -struct --- !query output -86399999 - - --- !query -SELECT time_to_millis(NULL) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_to_micros(TIME'00:00:00') --- !query schema -struct --- !query output -0 - - --- !query -SELECT time_to_micros(TIME'14:30:00') --- !query schema -struct --- !query output -52200000000 - - --- !query -SELECT time_to_micros(TIME'14:30:00.5') --- !query schema -struct --- !query output -52200500000 - - --- !query -SELECT time_to_micros(TIME'23:59:59.999') --- !query schema -struct --- !query output -86399999000 - - --- !query -SELECT time_to_micros(TIME'23:59:59.999999') --- !query schema -struct --- !query output -86399999999 - - --- !query -SELECT time_to_micros(NULL) --- !query schema -struct --- !query output -NULL - - --- !query -SELECT time_to_seconds(time_from_seconds(52200.5)) --- !query schema -struct --- !query output -52200.500000 - - --- !query -SELECT time_from_seconds(time_to_seconds(TIME'14:30:00.5')) --- !query schema -struct --- !query output -14:30:00.5 - - --- !query -SELECT time_to_millis(time_from_millis(52200500)) --- !query schema -struct --- !query output -52200500 - - --- !query -SELECT time_from_millis(time_to_millis(TIME'14:30:00.5')) --- !query schema -struct --- !query output -14:30:00.5 - - --- !query -SELECT time_to_micros(time_from_micros(52200500000)) --- !query schema -struct --- !query output -52200500000 - - --- !query -SELECT time_from_micros(time_to_micros(TIME'14:30:00.5')) --- !query schema -struct --- !query output -14:30:00.5 - - --- !query -SET spark.sql.ansi.enabled = true --- !query schema -struct --- !query output -spark.sql.ansi.enabled true - - -- !query SELECT time_from_seconds(0) -- !query schema From 50e6885c8b80eab54226d6f0813d1f3f5ad08ed3 Mon Sep 17 00:00:00 2001 From: vinodkc Date: Wed, 10 Dec 2025 20:53:58 -0800 Subject: [PATCH 5/5] Fix review comments 2 --- .../expressions/timeExpressions.scala | 153 ++++++++---------- .../sql/catalyst/util/DateTimeUtils.scala | 25 +-- .../sql-tests/analyzer-results/time.sql.out | 2 +- .../test/resources/sql-tests/inputs/time.sql | 15 +- .../resources/sql-tests/results/time.sql.out | 21 +-- 5 files changed, 90 insertions(+), 126 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala index 72bb55360de5..6469a70349a7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/timeExpressions.scala @@ -32,11 +32,22 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.catalyst.util.TimeFormatter import org.apache.spark.sql.catalyst.util.TypeUtils.ordinalNumber import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.types.StringTypeWithCollation import org.apache.spark.sql.types.{AbstractDataType, AnyTimeType, ByteType, DataType, DayTimeIntervalType, DecimalType, IntegerType, IntegralType, LongType, NumericType, ObjectType, TimeType} import org.apache.spark.sql.types.DayTimeIntervalType.{HOUR, SECOND} import org.apache.spark.unsafe.types.UTF8String +trait TimeExpression extends Expression { + override def checkInputDataTypes(): TypeCheckResult = { + if (SQLConf.get.isTimeTypeEnabled) { + super.checkInputDataTypes() + } else { + throw QueryCompilationErrors.unsupportedTimeTypeError() + } + } +} + /** * Parses a column to a time based on the given format. */ @@ -64,7 +75,7 @@ import org.apache.spark.unsafe.types.UTF8String since = "4.1.0") // scalastyle:on line.size.limit case class ToTime(str: Expression, format: Option[Expression]) - extends RuntimeReplaceable with ExpectsInputTypes { + extends RuntimeReplaceable with ExpectsInputTypes with TimeExpression { def this(str: Expression, format: Expression) = this(str, Option(format)) def this(str: Expression) = this(str, None) @@ -200,7 +211,7 @@ object TryToTimeExpressionBuilder extends ExpressionBuilder { // scalastyle:on line.size.limit case class MinutesOfTime(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes { + with ExpectsInputTypes with TimeExpression { override def replacement: Expression = StaticInvoke( classOf[DateTimeUtils.type], @@ -259,7 +270,7 @@ object MinuteExpressionBuilder extends ExpressionBuilder { case class HoursOfTime(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes { + with ExpectsInputTypes with TimeExpression { override def replacement: Expression = StaticInvoke( classOf[DateTimeUtils.type], @@ -316,7 +327,7 @@ object HourExpressionBuilder extends ExpressionBuilder { case class SecondsOfTimeWithFraction(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes { + with ExpectsInputTypes with TimeExpression { override def replacement: Expression = { val precision = child.dataType match { case TimeType(p) => p @@ -342,7 +353,7 @@ case class SecondsOfTimeWithFraction(child: Expression) case class SecondsOfTime(child: Expression) extends RuntimeReplaceable - with ExpectsInputTypes { + with ExpectsInputTypes with TimeExpression { override def replacement: Expression = StaticInvoke( classOf[DateTimeUtils.type], @@ -433,7 +444,8 @@ object SecondExpressionBuilder extends ExpressionBuilder { case class CurrentTime( child: Expression = Literal(TimeType.MICROS_PRECISION), timeZoneId: Option[String] = None) extends UnaryExpression - with TimeZoneAwareExpression with ImplicitCastInputTypes with CodegenFallback { + with TimeZoneAwareExpression with ImplicitCastInputTypes with CodegenFallback + with TimeExpression { def this() = { this(Literal(TimeType.MICROS_PRECISION), None) @@ -545,7 +557,7 @@ case class MakeTime( secsAndMicros: Expression) extends RuntimeReplaceable with ImplicitCastInputTypes - with ExpectsInputTypes { + with ExpectsInputTypes with TimeExpression { // Accept `sec` as DecimalType to avoid loosing precision of microseconds while converting // it to the fractional part of `sec`. If `sec` is an IntegerType, it can be cast into decimal @@ -570,7 +582,8 @@ case class MakeTime( * Adds day-time interval to time. */ case class TimeAddInterval(time: Expression, interval: Expression) - extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes { + extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes + with TimeExpression { override def nullIntolerant: Boolean = true override def left: Expression = time @@ -611,7 +624,8 @@ case class TimeAddInterval(time: Expression, interval: Expression) * Returns a day-time interval between time values. */ case class SubtractTimes(left: Expression, right: Expression) - extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes { + extends BinaryExpression with RuntimeReplaceable with ExpectsInputTypes + with TimeExpression { override def nullIntolerant: Boolean = true override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType, AnyTimeType) @@ -668,7 +682,8 @@ case class TimeDiff( end: Expression) extends TernaryExpression with RuntimeReplaceable - with ImplicitCastInputTypes { + with ImplicitCastInputTypes + with TimeExpression { override def first: Expression = unit override def second: Expression = start @@ -723,7 +738,8 @@ case class TimeDiff( since = "4.1.0") // scalastyle:on line.size.limit case class TimeTrunc(unit: Expression, time: Expression) - extends BinaryExpression with RuntimeReplaceable with ImplicitCastInputTypes { + extends BinaryExpression with RuntimeReplaceable with ImplicitCastInputTypes + with TimeExpression { override def left: Expression = unit override def right: Expression = time @@ -750,6 +766,22 @@ case class TimeTrunc(unit: Expression, time: Expression) } } +abstract class TimeFromBase extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes + with TimeExpression { + protected def timeConversionMethod: String + + override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) + override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) + + override def replacement: Expression = StaticInvoke( + classOf[DateTimeUtils.type], + dataType, + timeConversionMethod, + Seq(child), + Seq(child.dataType) + ) +} + // scalastyle:off line.size.limit @ExpressionDescription( usage = "_FUNC_(seconds) - Creates a TIME value from seconds since midnight.", @@ -772,20 +804,10 @@ case class TimeTrunc(unit: Expression, time: Expression) since = "4.2.0", group = "datetime_funcs") // scalastyle:on line.size.limit -case class TimeFromSeconds(child: Expression) - extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - - override def replacement: Expression = StaticInvoke( - classOf[DateTimeUtils.type], - TimeType(TimeType.MICROS_PRECISION), - "timeFromSeconds", - Seq(child), - Seq(child.dataType) - ) - +case class TimeFromSeconds(child: Expression) extends TimeFromBase { override def inputTypes: Seq[AbstractDataType] = Seq(NumericType) - override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) override def prettyName: String = "time_from_seconds" + override protected def timeConversionMethod: String = "timeFromSeconds" override protected def withNewChildInternal(newChild: Expression): TimeFromSeconds = copy(child = newChild) @@ -812,20 +834,9 @@ case class TimeFromSeconds(child: Expression) since = "4.2.0", group = "datetime_funcs") // scalastyle:on line.size.limit -case class TimeFromMillis(child: Expression) - extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - - override def replacement: Expression = StaticInvoke( - classOf[DateTimeUtils.type], - TimeType(TimeType.MICROS_PRECISION), - "timeFromMillis", - Seq(child), - Seq(child.dataType) - ) - - override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) - override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) +case class TimeFromMillis(child: Expression) extends TimeFromBase { override def prettyName: String = "time_from_millis" + override protected def timeConversionMethod: String = "timeFromMillis" override protected def withNewChildInternal(newChild: Expression): TimeFromMillis = copy(child = newChild) @@ -852,23 +863,28 @@ case class TimeFromMillis(child: Expression) since = "4.2.0", group = "datetime_funcs") // scalastyle:on line.size.limit -case class TimeFromMicros(child: Expression) - extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { +case class TimeFromMicros(child: Expression) extends TimeFromBase { + override def prettyName: String = "time_from_micros" + override protected def timeConversionMethod: String = "timeFromMicros" + + override protected def withNewChildInternal(newChild: Expression): TimeFromMicros = + copy(child = newChild) +} + +abstract class TimeToBase extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes + with TimeExpression { + protected def timeConversionMethod: String + + override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) + override def dataType: DataType = LongType override def replacement: Expression = StaticInvoke( classOf[DateTimeUtils.type], - TimeType(TimeType.MICROS_PRECISION), - "timeFromMicros", + dataType, + timeConversionMethod, Seq(child), Seq(child.dataType) ) - - override def inputTypes: Seq[AbstractDataType] = Seq(IntegralType) - override def dataType: DataType = TimeType(TimeType.MICROS_PRECISION) - override def prettyName: String = "time_from_micros" - - override protected def withNewChildInternal(newChild: Expression): TimeFromMicros = - copy(child = newChild) } // scalastyle:off line.size.limit @@ -893,20 +909,11 @@ case class TimeFromMicros(child: Expression) since = "4.2.0", group = "datetime_funcs") // scalastyle:on line.size.limit -case class TimeToSeconds(child: Expression) - extends UnaryExpression with RuntimeReplaceable with ImplicitCastInputTypes { - - override def replacement: Expression = StaticInvoke( - classOf[DateTimeUtils.type], - DecimalType(14, 6), - "timeToSeconds", - Seq(child), - Seq(child.dataType) - ) +case class TimeToSeconds(child: Expression) extends TimeToBase { - override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) override def dataType: DataType = DecimalType(14, 6) override def prettyName: String = "time_to_seconds" + override protected def timeConversionMethod: String = "timeToSeconds" override protected def withNewChildInternal(newChild: Expression): TimeToSeconds = copy(child = newChild) @@ -934,20 +941,9 @@ case class TimeToSeconds(child: Expression) since = "4.2.0", group = "datetime_funcs") // scalastyle:on line.size.limit -case class TimeToMillis(child: Expression) - extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - - override def replacement: Expression = StaticInvoke( - classOf[DateTimeUtils.type], - LongType, - "timeToMillis", - Seq(child), - Seq(child.dataType) - ) - - override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) - override def dataType: DataType = LongType +case class TimeToMillis(child: Expression) extends TimeToBase { override def prettyName: String = "time_to_millis" + override protected def timeConversionMethod: String = "timeToMillis" override protected def withNewChildInternal(newChild: Expression): TimeToMillis = copy(child = newChild) @@ -975,20 +971,9 @@ case class TimeToMillis(child: Expression) since = "4.2.0", group = "datetime_funcs") // scalastyle:on line.size.limit -case class TimeToMicros(child: Expression) - extends UnaryExpression with RuntimeReplaceable with ExpectsInputTypes { - - override def replacement: Expression = StaticInvoke( - classOf[DateTimeUtils.type], - LongType, - "timeToMicros", - Seq(child), - Seq(child.dataType) - ) - - override def inputTypes: Seq[AbstractDataType] = Seq(AnyTimeType) - override def dataType: DataType = LongType +case class TimeToMicros(child: Expression) extends TimeToBase { override def prettyName: String = "time_to_micros" + override protected def timeConversionMethod: String = "timeToMicros" override protected def withNewChildInternal(newChild: Expression): TimeToMicros = copy(child = newChild) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala index 6739d640e68e..82072443ec0a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala @@ -908,10 +908,10 @@ object DateTimeUtils extends SparkDateTimeUtils { nanos } catch { case e: DateTimeException => - throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(e) + throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRangeWithoutSuggestion(e) case e: ArithmeticException => - val wrapped = new DateTimeException(s"Overflow in TIME conversion: ${e.getMessage}", e) - throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRange(wrapped) + throw QueryExecutionErrors.ansiDateTimeArgumentOutOfRangeWithoutSuggestion( + new DateTimeException("Overflow in TIME conversion", e)) } } @@ -935,19 +935,7 @@ object DateTimeUtils extends SparkDateTimeUtils { } /** - * Creates a TIME value from seconds since midnight (float type). - * @param seconds Seconds (0 to 86399.999999) - * @return Nanoseconds since midnight - */ - def timeFromSeconds(seconds: Float): Long = withTimeConversionErrorHandling { - if (seconds.isNaN || seconds.isInfinite) { - throw new DateTimeException("Cannot convert NaN or Infinite value to TIME") - } - (seconds.toDouble * NANOS_PER_SECOND).toLong - } - - /** - * Creates a TIME value from seconds since midnight (double type). + * Creates a TIME value from seconds since midnight (floating point type). * @param seconds Seconds (0 to 86399.999999) * @return Nanoseconds since midnight */ @@ -983,10 +971,7 @@ object DateTimeUtils extends SparkDateTimeUtils { */ def timeToSeconds(nanos: Long): Decimal = { val result = Decimal(nanos) / Decimal(NANOS_PER_SECOND) - if (!result.changePrecision(14, 6)) { - throw new DateTimeException( - "TIME to seconds conversion resulted in value that cannot fit in Decimal(14, 6)") - } + result.changePrecision(14, 6) result } diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out index 7075a9f8c4b4..8c5c55bfa0a0 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/time.sql.out @@ -2166,7 +2166,7 @@ Project [time_to_seconds(23:59:59.999999) AS time_to_seconds(TIME '23:59:59.9999 -- !query SELECT time_to_seconds(NULL) -- !query analysis -Project [time_to_seconds(cast(null as time(6))) AS time_to_seconds(NULL)#x] +Project [time_to_seconds(null) AS time_to_seconds(NULL)#x] +- OneRowRelation diff --git a/sql/core/src/test/resources/sql-tests/inputs/time.sql b/sql/core/src/test/resources/sql-tests/inputs/time.sql index a0e1ba5973be..f81944716881 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/time.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/time.sql @@ -304,14 +304,15 @@ SELECT '00:00:00.1234' :: TIME(4) - TIME'23:59:59'; -- Numeric constructor and extractor functions for TIME type + -- time_from_seconds (valid: 0 to 86399.999999) SELECT time_from_seconds(0); SELECT time_from_seconds(43200); SELECT time_from_seconds(52200.5); SELECT time_from_seconds(86399.999999); -SELECT time_from_seconds(-1); -- invalid: negative → exception -SELECT time_from_seconds(86400); -- invalid: >= 86400 → exception -SELECT time_from_seconds(90000); -- invalid: >= 86400 → exception +SELECT time_from_seconds(-1); -- invalid: negative -> exception +SELECT time_from_seconds(86400); -- invalid: >= 86400 -> exception +SELECT time_from_seconds(90000); -- invalid: >= 86400 -> exception SELECT time_from_seconds(NULL); -- time_from_millis (valid: 0 to 86399999) @@ -320,8 +321,8 @@ SELECT time_from_millis(43200); SELECT time_from_millis(52200000); SELECT time_from_millis(52200500); SELECT time_from_millis(86399999); -SELECT time_from_millis(-1); -- invalid: negative → exception -SELECT time_from_millis(86400000); -- invalid: >= 86400000 → exception +SELECT time_from_millis(-1); -- invalid: negative -> exception +SELECT time_from_millis(86400000); -- invalid: >= 86400000 -> exception SELECT time_from_millis(NULL); -- time_from_micros (valid: 0 to 86399999999) @@ -330,8 +331,8 @@ SELECT time_from_micros(43200); SELECT time_from_micros(52200000000); SELECT time_from_micros(52200500000); SELECT time_from_micros(86399999999); -SELECT time_from_micros(-1); -- invalid: negative → exception -SELECT time_from_micros(86400000000); -- invalid: >= 86400000000 → exception +SELECT time_from_micros(-1); -- invalid: negative -> exception +SELECT time_from_micros(86400000000); -- invalid: >= 86400000000 -> exception SELECT time_from_micros(NULL); -- time_to_seconds diff --git a/sql/core/src/test/resources/sql-tests/results/time.sql.out b/sql/core/src/test/resources/sql-tests/results/time.sql.out index 033f9be598f2..5767be09ece5 100644 --- a/sql/core/src/test/resources/sql-tests/results/time.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/time.sql.out @@ -2418,10 +2418,9 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION", "sqlState" : "22023", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got -1000000000 nanoseconds" } } @@ -2434,10 +2433,9 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION", "sqlState" : "22023", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 86400000000000 nanoseconds" } } @@ -2450,10 +2448,9 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION", "sqlState" : "22023", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 90000000000000 nanoseconds" } } @@ -2514,10 +2511,9 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION", "sqlState" : "22023", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got -1000000 nanoseconds" } } @@ -2530,10 +2526,9 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION", "sqlState" : "22023", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 86400000000000 nanoseconds" } } @@ -2594,10 +2589,9 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION", "sqlState" : "22023", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got -1000 nanoseconds" } } @@ -2610,10 +2604,9 @@ struct<> -- !query output org.apache.spark.SparkDateTimeException { - "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITH_SUGGESTION", + "errorClass" : "DATETIME_FIELD_OUT_OF_BOUNDS.WITHOUT_SUGGESTION", "sqlState" : "22023", "messageParameters" : { - "ansiConfig" : "\"spark.sql.ansi.enabled\"", "rangeMessage" : "Invalid TIME value: must be between 00:00:00 and 23:59:59.999999999, but got 86400000000000 nanoseconds" } }