From eb9089a89f9fbbfc255f039d37e4b1974d8f5a0b Mon Sep 17 00:00:00 2001 From: "pavel.voropaev" Date: Tue, 18 Apr 2023 10:32:36 +0100 Subject: [PATCH] Use 'Numeric' instead of 'Float' as BQ column type when field type in the json schema is number (close #112) --- .../iglu.schemaddl/bigquery/Row.scala | 2 +- .../iglu.schemaddl/bigquery/Suggestion.scala | 72 ++++++++++++++++-- .../iglu.schemaddl/bigquery/Type.scala | 18 ++++- .../jsonschema/suggestion/decimals.scala | 73 ++++++++++++++++++ .../jsonschema/suggestion/numericType.scala | 25 +++++++ .../iglu.schemaddl/parquet/Field.scala | 24 ++++-- .../iglu.schemaddl/parquet/Suggestion.scala | 68 ++--------------- .../iglu.schemaddl/parquet/Type.scala | 43 ++++++++--- .../iglu/schemaddl/bigquery/FieldSpec.scala | 74 ++++++++++++++++--- .../iglu/schemaddl/bigquery/RowSpec.scala | 2 +- 10 files changed, 305 insertions(+), 96 deletions(-) create mode 100644 modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala create mode 100644 modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala index 0b751434..f0bbc4cb 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Row.scala @@ -59,7 +59,7 @@ object Row { value.asNumber.flatMap(_.toLong).fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) case Type.Float => value.asNumber.flatMap(_.toBigDecimal.map(_.bigDecimal)).fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) - case Type.Numeric => + case Type.Numeric(_, _) => value.asNumber.flatMap(_.toBigDecimal.map(_.bigDecimal)).fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) case Type.Timestamp => value.asString.fold(WrongType(value, fieldType).invalidNel[Row])(Primitive(_).validNel) diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala index 8a90141f..fab73a58 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Suggestion.scala @@ -13,9 +13,10 @@ package com.snowplowanalytics.iglu.schemaddl.bigquery import io.circe._ - import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema -import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{CommonProperties, StringProperty} +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.decimals +import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{CommonProperties, StringProperty, NumberProperty} +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType.NullableWrapper object Suggestion { @@ -24,7 +25,7 @@ object Suggestion { case Some(CommonProperties.Type.String) => Some(name => Field(name, Type.String, Mode.required(required))) case Some(types) if types.nullable(CommonProperties.Type.String) => - Some(name => Field(name, Type.String, Mode.Nullable) ) + Some(name => Field(name, Type.String, Mode.Nullable)) case _ => None } @@ -40,19 +41,55 @@ object Suggestion { val integerSuggestion: Suggestion = (schema, required) => schema.`type` match { case Some(CommonProperties.Type.Integer) => - Some(name => Field(name, Type.Integer, Mode.required(required))) + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.required(required))) case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Integer) => - Some(name => Field(name, Type.Integer, Mode.Nullable)) + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable)) case _ => None } + val numericSuggestion: Suggestion = (schema, required) => schema.`type` match { + case Some(CommonProperties.Type.Number) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.required(required))) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.required(required))) + case None => None + } + case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Number) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable)) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.Nullable)) + case None => None + } + case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, true) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.Nullable)) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.Nullable)) + case None => None + } + case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, false) => + schema.multipleOf match { + case Some(NumberProperty.MultipleOf.IntegerMultipleOf(_)) => + Some(name => Field(name, Type.fromGenericType(decimals.integerType(schema)), Mode.required(required))) + case Some(mult: NumberProperty.MultipleOf.NumberMultipleOf) => + Some(name => Field(name, numericWithMultiple(mult, schema.maximum, schema.minimum), Mode.required(required))) + case None => None + } + case _ => None + } + val floatSuggestion: Suggestion = (schema, required) => schema.`type` match { case Some(CommonProperties.Type.Number) => Some(name => Field(name, Type.Float, Mode.required(required))) case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, true) => Some(name => Field(name, Type.Float, Mode.Nullable)) - case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, false) => + case Some(CommonProperties.Type.Union(types)) if onlyNumeric(types, false) => Some(name => Field(name, Type.Float, Mode.required(required))) case Some(CommonProperties.Type.Union(types)) if withNull(types, CommonProperties.Type.Number) => Some(name => Field(name, Type.Float, Mode.Nullable)) @@ -66,6 +103,15 @@ object Suggestion { case _ => None } + private def numericWithMultiple(mult: NumberProperty.MultipleOf.NumberMultipleOf, + maximum: Option[NumberProperty.Maximum], + minimum: Option[NumberProperty.Minimum]): Type = + Type.fromGenericType(decimals.numericWithMultiple(mult, maximum, minimum)) + + + // (Field.JsonNullability.fromNullableWrapper) + + // `date-time` format usually means zoned format, which corresponds to BQ Timestamp val timestampSuggestion: Suggestion = (schema, required) => (schema.`type`, schema.format) match { @@ -95,14 +141,18 @@ object Suggestion { booleanSuggestion, stringSuggestion, integerSuggestion, + numericSuggestion, floatSuggestion, complexEnumSuggestion ) private[iglu] def fromEnum(enums: List[Json], required: Boolean): String => Field = { def isString(json: Json) = json.isString || json.isNull + def isInteger(json: Json) = json.asNumber.exists(_.toBigInt.isDefined) || json.isNull + def isNumeric(json: Json) = json.isNumber || json.isNull + val noNull: Boolean = !enums.contains(Json.Null) if (enums.forall(isString)) { @@ -110,7 +160,15 @@ object Suggestion { } else if (enums.forall(isInteger)) { name => Field(name, Type.Integer, Mode.required(required && noNull)) } else if (enums.forall(isNumeric)) { - name => Field(name, Type.Float, Mode.required(required && noNull)) + name => + decimals.numericEnum(enums).map { + case NullableWrapper.NullableValue(t) => Field(name, Type.fromGenericType(t), Mode.required(required && noNull)) + case NullableWrapper.NotNullValue(t) => Field(name, Type.fromGenericType(t), Mode.required(required && noNull)) + } match { + case Some(value) => value + // Unreachable as `None` here would mean that some `enums.forall(isNumeric)` did not work. + case None => Field(name, Type.Float, Mode.required(required && noNull)) + } } else { name => Field(name, Type.String, Mode.required(required && noNull)) } diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala index 8fc8c3fd..f4ddaa00 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/bigquery/Type.scala @@ -12,18 +12,34 @@ */ package com.snowplowanalytics.iglu.schemaddl.bigquery +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType._ /** BigQuery field type; "array" and "null" are expressed via `Mode` */ sealed trait Type extends Product with Serializable object Type { case object String extends Type + case object Boolean extends Type + case object Integer extends Type + case object Float extends Type - case object Numeric extends Type + + case class Numeric(precision: Int, scale: Int) extends Type + case object Date extends Type + case object DateTime extends Type + case object Timestamp extends Type + case class Record(fields: List[Field]) extends Type + + def fromGenericType(`type`: NumericType) = `type` match { + case NumericType.Double => Float + case NumericType.Int32 => Integer + case NumericType.Int64 => Integer + case NumericType.Decimal(precision, scale) => Numeric(precision, scale) + } } diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala new file mode 100644 index 00000000..cf2edb16 --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/decimals.scala @@ -0,0 +1,73 @@ +package com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion + +import numericType._ +import io.circe.Json +import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.NumberProperty +import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema + +private[schemaddl] object decimals { + + def integerType(schema: Schema): NumericType = + (schema.minimum, schema.maximum) match { + case (Some(min), Some(max)) => + val minDecimal = min.getAsDecimal + val maxDecimal = max.getAsDecimal + if (maxDecimal <= Int.MaxValue && minDecimal >= Int.MinValue) NumericType.Int32 + else if (maxDecimal <= Long.MaxValue && minDecimal >= Long.MinValue) NumericType.Int64 + else NumericType.Decimal( + (maxDecimal.precision - maxDecimal.scale).max(minDecimal.precision - minDecimal.scale), 0 + ) + case _ => NumericType.Int64 + } + + def numericWithMultiple(mult: NumberProperty.MultipleOf.NumberMultipleOf, + maximum: Option[NumberProperty.Maximum], + minimum: Option[NumberProperty.Minimum]): NumericType = + (maximum, minimum) match { + case (Some(max), Some(min)) => + val topPrecision = max match { + case NumberProperty.Maximum.IntegerMaximum(max) => + BigDecimal(max).precision + mult.value.scale + case NumberProperty.Maximum.NumberMaximum(max) => + max.precision - max.scale + mult.value.scale + } + val bottomPrecision = min match { + case NumberProperty.Minimum.IntegerMinimum(min) => + BigDecimal(min).precision + mult.value.scale + case NumberProperty.Minimum.NumberMinimum(min) => + min.precision - min.scale + mult.value.scale + } + + NumericType.Decimal(topPrecision.max(bottomPrecision), mult.value.scale) + + case _ => + NumericType.Double + } + + + def numericEnum(enums: List[Json]): Option[NullableWrapper] = { + def go(scale: Int, max: BigDecimal, nullable: Boolean, enums: List[Json]): Option[NullableWrapper] = + enums match { + case Nil => + val t = if ((scale == 0) && (max <= Int.MaxValue)) NumericType.Int32 + else if ((scale == 0) && (max <= Long.MaxValue)) NumericType.Int64 + else NumericType.Decimal(max.precision - max.scale + scale, scale) + + Some(if (nullable) NullableWrapper.NullableValue(t) + else NullableWrapper.NotNullValue(t)) + + case Json.Null :: tail => go(scale, max, true, tail) + case h :: tail => + h.asNumber.flatMap(_.toBigDecimal) match { + case Some(bigDecimal) => + val nextScale = scale.max(bigDecimal.scale) + val nextMax = (if (bigDecimal > 0) bigDecimal else -bigDecimal).max(max) + go(nextScale, nextMax, nullable, tail) + case None => None + } + } + + go(0, 0, false, enums) + } + +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala new file mode 100644 index 00000000..d2496a33 --- /dev/null +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/jsonschema/suggestion/numericType.scala @@ -0,0 +1,25 @@ +package com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion + +private[schemaddl] object numericType { + sealed trait NumericType extends Product with Serializable + + object NumericType { + case object Double extends NumericType + + case object Int32 extends NumericType + + case object Int64 extends NumericType + + case class Decimal(precision: Int, scale: Int) extends NumericType + } + + sealed trait NullableWrapper + + object NullableWrapper { + + case class NullableValue(value: NumericType) extends NullableWrapper + + case class NotNullValue(value: NumericType) extends NullableWrapper + + } +} diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala index 614c4cda..044152fb 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Field.scala @@ -12,13 +12,15 @@ */ package com.snowplowanalytics.iglu.schemaddl.parquet +import Type._ import com.snowplowanalytics.iglu.schemaddl.StringUtils import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{ArrayProperty, CommonProperties} import com.snowplowanalytics.iglu.schemaddl.jsonschema.mutate.Mutate +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType.NullableWrapper -case class Field(name: String, - fieldType: Type, +case class Field(name: String, + fieldType: Type, nullability: Type.Nullability) object Field { @@ -67,8 +69,20 @@ object Field { private[parquet] object JsonNullability { case object ExplicitlyNullable extends JsonNullability + case object NoExplicitNull extends JsonNullability + def fromNullableWrapper(wrapper: NullableWrapper): NullableType = wrapper match { + case NullableWrapper.NullableValue(t) => NullableType( + value = fromGenericType(t), + nullability = JsonNullability.ExplicitlyNullable + ) + case NullableWrapper.NotNullValue(t) => NullableType( + value = fromGenericType(t), + nullability = JsonNullability.NoExplicitNull + ) + } + def extractFrom(`type`: CommonProperties.Type): JsonNullability = { if (`type`.nullable) { JsonNullability.ExplicitlyNullable @@ -81,13 +95,13 @@ object Field { topSchema.`type` match { case Some(types) if types.possiblyWithNull(CommonProperties.Type.Object) => NullableType( - value = buildObjectType(topSchema), + value = buildObjectType(topSchema), nullability = JsonNullability.extractFrom(types) ) case Some(types) if types.possiblyWithNull(CommonProperties.Type.Array) => - NullableType( - value = buildArrayType(topSchema), + NullableType( + value = buildArrayType(topSchema), nullability = JsonNullability.extractFrom(types) ) diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala index eab58ffa..1bd199cd 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Suggestion.scala @@ -12,8 +12,8 @@ */ package com.snowplowanalytics.iglu.schemaddl.parquet -import cats.implicits._ import com.snowplowanalytics.iglu.schemaddl.jsonschema.Schema +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.decimals import com.snowplowanalytics.iglu.schemaddl.jsonschema.properties.{CommonProperties, NumberProperty, StringProperty} import io.circe._ @@ -80,55 +80,9 @@ private[parquet] object Suggestion { private def numericWithMultiple(mult: NumberProperty.MultipleOf.NumberMultipleOf, maximum: Option[NumberProperty.Maximum], minimum: Option[NumberProperty.Minimum]): Type = - (maximum, minimum) match { - case (Some(max), Some(min)) => - val topPrecision = max match { - case NumberProperty.Maximum.IntegerMaximum(max) => - BigDecimal(max).precision + mult.value.scale - case NumberProperty.Maximum.NumberMaximum(max) => - max.precision - max.scale + mult.value.scale - } - val bottomPrecision = min match { - case NumberProperty.Minimum.IntegerMinimum(min) => - BigDecimal(min).precision + mult.value.scale - case NumberProperty.Minimum.NumberMinimum(min) => - min.precision - min.scale + mult.value.scale - } - Type.DecimalPrecision.of(topPrecision.max(bottomPrecision)) match { - case Some(precision) => - Type.Decimal(precision, mult.value.scale) - case None => - Type.Double - } - case _ => - Type.Double - } + Type.fromGenericType(decimals.numericWithMultiple(mult, maximum, minimum)) - - private def numericEnum(enums: List[Json]): Option[Field.NullableType] = { - def go(scale: Int, max: BigDecimal, nullable: Field.JsonNullability, enums: List[Json]): Option[Field.NullableType] = - enums match { - case Nil => - val t = if (scale === 0 && max <= Int.MaxValue) Type.Integer - else if (scale === 0 && max <= Long.MaxValue) Type.Long - else { - val precision = (max.precision - max.scale) + scale - Type.DecimalPrecision.of(precision).fold[Type](Type.Double)(Type.Decimal(_, scale)) - } - Some(Field.NullableType(t, nullable)) - case Json.Null :: tail => go(scale, max, Field.JsonNullability.ExplicitlyNullable, tail) - case h :: tail => - h.asNumber.flatMap(_.toBigDecimal) match { - case Some(bigDecimal) => - val nextScale = scale.max(bigDecimal.scale) - val nextMax = (if (bigDecimal > 0) bigDecimal else -bigDecimal).max(max) - go(nextScale, nextMax, nullable, tail) - case None => None - } - } - - go(0, 0, Field.JsonNullability.NoExplicitNull, enums) - } + private def numericEnum(enums: List[Json]): Option[Field.NullableType] = decimals.numericEnum(enums).map(Field.JsonNullability.fromNullableWrapper) private def stringEnum(enums: List[Json]): Option[Field.NullableType] = { def go(nullable: Field.JsonNullability, enums: List[Json]): Option[Field.NullableType] = @@ -138,6 +92,7 @@ private[parquet] object Suggestion { case h :: tail if h.isString => go(nullable, tail) case _ => None } + go(Field.JsonNullability.NoExplicitNull, enums) } @@ -146,19 +101,8 @@ private[parquet] object Suggestion { Field.NullableType(Type.Json, nullable) } - private def integerType(schema: Schema): Type = - (schema.minimum, schema.maximum) match { - case (Some(min), Some(max)) => - val minDecimal = min.getAsDecimal - val maxDecimal = max.getAsDecimal - if (maxDecimal <= Int.MaxValue && minDecimal >= Int.MinValue) Type.Integer - else if (maxDecimal <= Long.MaxValue && minDecimal >= Long.MinValue) Type.Long - else Type.DecimalPrecision - .of((maxDecimal.precision - maxDecimal.scale).max(minDecimal.precision - minDecimal.scale)) - .fold[Type](Type.Double)(Type.Decimal(_, 0)) - case _ => Type.Long - } - + private def integerType(schema: Schema): Type = Type.fromGenericType(decimals.integerType(schema)) + private def onlyNumeric(types: CommonProperties.Type): Boolean = types match { case CommonProperties.Type.Number => true diff --git a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala index 0ecd94fa..1d00bc57 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics/iglu.schemaddl/parquet/Type.scala @@ -13,20 +13,30 @@ package com.snowplowanalytics.iglu.schemaddl.parquet import cats.Eq +import com.snowplowanalytics.iglu.schemaddl.jsonschema.suggestion.numericType._ sealed trait Type extends Product with Serializable object Type { - case object String extends Type - case object Boolean extends Type - case object Integer extends Type - case object Long extends Type - case object Double extends Type - case class Decimal(precision: DecimalPrecision, scale: Int) extends Type - case object Date extends Type - case object Timestamp extends Type - case class Struct(fields: List[Field]) extends Type + case object String extends Type + + case object Boolean extends Type + + case object Integer extends Type + + case object Long extends Type + + case object Double extends Type + + case class Decimal(precision: DecimalPrecision, scale: Int) extends Type + + case object Date extends Type + + case object Timestamp extends Type + + case class Struct(fields: List[Field]) extends Type + case class Array(element: Type, nullability: Nullability) extends Type /* Fallback type for when json schema does not map to a parquet primitive type (e.g. unions) @@ -41,6 +51,7 @@ object Type { sealed trait Nullability { def nullable: Boolean + def required: Boolean = !nullable } @@ -48,15 +59,19 @@ object Type { case object Nullable extends Nullability { override def nullable: Boolean = true } + case object Required extends Nullability { override def nullable: Boolean = false } } sealed trait DecimalPrecision + object DecimalPrecision { case object Digits9 extends DecimalPrecision // Int32 physical type + case object Digits18 extends DecimalPrecision // Int64 physical type + case object Digits38 extends DecimalPrecision // Fixed length byte array physical type. def of(precision: Int): Option[DecimalPrecision] = @@ -71,4 +86,14 @@ object Type { case Digits38 => 38 } } + + def fromGenericType(`type`: NumericType) = `type` match { + case NumericType.Double => Double + case NumericType.Int32 => Integer + case NumericType.Int64 => Long + case NumericType.Decimal(precision, scale) => DecimalPrecision.of(precision) match { + case Some(value) => Decimal(value, scale) + case None => Double + } + } } diff --git a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala index f6cb9acb..1c22df1d 100644 --- a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala +++ b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/FieldSpec.scala @@ -26,6 +26,8 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" build generates nullable field for oneOf types $e10 build generates nullable field for nullable object without nested keys $e11 build generates nullable field for nullable array without items $e12 + build generates numeric/decimal for enums $e13 + build generates numeric/decimal for multipleof $e14 """ def e1 = { @@ -61,7 +63,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" )), Mode.Nullable ), - Field("stringKey", Type.String,Mode.Nullable))), + Field("stringKey", Type.String, Mode.Nullable))), Mode.Nullable ) @@ -94,7 +96,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" ) Field.build("foo", input, false) must beEqualTo(expected) - } + } def e3 = { val input = SpecHelpers.parseSchema( @@ -159,9 +161,9 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" |} """.stripMargin) - val expected = Field("foo",Type.Record(List( - Field("union",Type.String,Mode.Nullable) - )),Mode.Nullable) + val expected = Field("foo", Type.Record(List( + Field("union", Type.String, Mode.Nullable) + )), Mode.Nullable) Field.build("foo", input, false) must beEqualTo(expected) } @@ -178,9 +180,9 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" |} """.stripMargin) - val expected = Field("foo",Type.Record(List( - Field("union",Type.String,Mode.Nullable) - )),Mode.Nullable) + val expected = Field("foo", Type.Record(List( + Field("union", Type.String, Mode.Nullable) + )), Mode.Nullable) Field.build("foo", input, false) must beEqualTo(expected) } @@ -199,7 +201,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" | } """.stripMargin) - val expected = Field("arrayTest",Type.Record(List(Field("imp",Type.String,Mode.Repeated))),Mode.Required) + val expected = Field("arrayTest", Type.Record(List(Field("imp", Type.String, Mode.Repeated))), Mode.Required) Field.build("arrayTest", input, true) must beEqualTo(expected) } @@ -217,7 +219,7 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" | } """.stripMargin) - val expected = Field("arrayTest",Type.Record(List(Field("imp",Type.String,Mode.Repeated))),Mode.Required) + val expected = Field("arrayTest", Type.Record(List(Field("imp", Type.String, Mode.Repeated))), Mode.Required) Field.build("arrayTest", input, true) must beEqualTo(expected) } @@ -313,5 +315,57 @@ class FieldSpec extends org.specs2.Specification { def is = s2""" Field.build("foo", input, false) must beEqualTo(expected) } + def e13 = { + val input = SpecHelpers.parseSchema( + """ + | { + | "type": "object", + | "required": ["xyz"], + | "properties": { + | "xyz": { + | "enum": [10, 1.12, 1e9] + | } + | } + | } + """.stripMargin) + + val expected = Field( + "foo", + Type.Record(List( + Field("xyz", Type.Numeric(12,2), Mode.Required) + )), + Mode.Nullable + ) + + Field.build("foo", input, false) must beEqualTo(expected) + } + + def e14 = { + val input = SpecHelpers.parseSchema( + """ + | { + | "type": "object", + | "required": ["xyz"], + | "properties": { + | "xyz": { + | "type": ["number", "null"], + | "multipleOf": 0.001, + | "maximum": 2, + | "minimum": 1 + | } + | } + | } + """.stripMargin) + + val expected = Field( + "foo", + Type.Record(List( + Field("xyz", Type.Numeric(4,3), Mode.Nullable) + )), + Mode.Nullable + ) + + Field.build("foo", input, false) must beEqualTo(expected) + } private def fieldNormalName(name: String) = Field(name, Type.String, Mode.Nullable).normalName } diff --git a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala index c9a99471..73d7ee2a 100644 --- a/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala +++ b/modules/core/src/test/scala/com/snowplowanalytics/iglu/schemaddl/bigquery/RowSpec.scala @@ -36,7 +36,7 @@ class RowSpec extends org.specs2.Specification { def is = s2""" def e1 = { val string = castValue(Type.String)(json""""foo"""") must beValid(Primitive("foo")) val int = castValue(Type.Integer)(json"-43") must beValid(Primitive(-43)) - val num = castValue(Type.Numeric)(json"-87.98") must beValid(Primitive(new java.math.BigDecimal("-87.98"))) + val num = castValue(Type.Numeric(1,0))(json"-87.98") must beValid(Primitive(new java.math.BigDecimal("-87.98"))) val bool = castValue(Type.Boolean)(Json.fromBoolean(false)) must beValid(Primitive(false)) string and int and num and bool }