From 94e9497fa8fc98aa3f6f9a8b6be83ce7ede721bf Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Sat, 18 Nov 2023 20:24:55 +0000 Subject: [PATCH 1/5] Null support, ignore delimiters start/end of line --- .../indigoplugin/generators/EmbedData.scala | 64 ++++++++++++++++--- .../generators/EmbedDataTests.scala | 17 +++-- 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala index c7483e08e..523504100 100644 --- a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala +++ b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala @@ -74,16 +74,35 @@ object EmbedData { Seq(file) } - def extractRows(rows: List[String], delimiter: String): List[List[DataType]] = - rows.map(r => extractRowData(r, delimiter)) + def extractRowData(row: String, delimiter: String): List[DataType] = { - def extractRowData(row: String, delimiter: String): List[DataType] = - parse(delimiter)(row).map(_._1).collect { + val cleanDelimiter: String = + if (delimiter == "\\|") "|" else delimiter + + val cleanRow: String = + row.trim match { + case r if r.startsWith(cleanDelimiter) && r.endsWith(cleanDelimiter) => + r.drop(cleanDelimiter.length()).dropRight(cleanDelimiter.length()) + + case r if r.startsWith(cleanDelimiter) => + r.drop(cleanDelimiter.length()) + + case r if r.endsWith(cleanDelimiter) => + r.dropRight(cleanDelimiter.length()) + + case r => + r + } + + parse(delimiter)(cleanRow).map(_._1).collect { case d @ DataType.StringData(s) if s.nonEmpty => d + case DataType.StringData(_) => DataType.NullData case d: DataType.BooleanData => d case d: DataType.DoubleData => d case d: DataType.IntData => d + case DataType.NullData => DataType.NullData } + } // A parser of things, // is a function from strings, @@ -137,12 +156,19 @@ sealed trait DataType { case _ => false } + def isNull: Boolean = + this match { + case DataType.NullData => true + case _ => false + } + def toStringData: DataType.StringData = this match { case s: DataType.StringData => s case DataType.BooleanData(value) => DataType.StringData(value.toString) case DataType.DoubleData(value) => DataType.StringData(value.toString) case DataType.IntData(value) => DataType.StringData(value.toString) + case DataType.NullData => DataType.StringData("null") } def asString: String = @@ -151,6 +177,7 @@ sealed trait DataType { case DataType.BooleanData(value) => value.toString case DataType.DoubleData(value) => value.toString case DataType.IntData(value) => value.toString + case DataType.NullData => "null" } def giveTypeName: String = @@ -159,6 +186,7 @@ sealed trait DataType { case _: DataType.BooleanData => "Boolean" case _: DataType.DoubleData => "Double" case _: DataType.IntData => "Int" + case DataType.NullData => "Null" } } @@ -171,24 +199,35 @@ object DataType { } final case class DoubleData(value: Double) extends DataType final case class StringData(value: String) extends DataType + case object NullData extends DataType private val isBoolean: Regex = """^(true|false)$""".r private val isDouble: Regex = """^([0-9]+?)\.([0-9]+)$""".r private val isInt: Regex = """^([0-9]+)$""".r + private val isNull: Regex = """^(null)$""".r def decideType: String => DataType = { case isBoolean(v) => BooleanData(v.toBoolean) case isInt(v) => IntData(v.toInt) case isDouble(v1, v2) => DoubleData(s"$v1.$v2".toDouble) + case isNull(_) => NullData case v => StringData(v) } def sameType(a: DataType, b: DataType): Boolean = (a, b) match { case (_: DataType.StringData, _: DataType.StringData) => true + case (DataType.NullData, _: DataType.StringData) => true + case (_: DataType.StringData, DataType.NullData) => true case (_: DataType.BooleanData, _: DataType.BooleanData) => true + case (DataType.NullData, _: DataType.BooleanData) => true + case (_: DataType.BooleanData, DataType.NullData) => true case (_: DataType.DoubleData, _: DataType.DoubleData) => true + case (DataType.NullData, _: DataType.DoubleData) => true + case (_: DataType.DoubleData, DataType.NullData) => true case (_: DataType.IntData, _: DataType.IntData) => true + case (DataType.NullData, _: DataType.IntData) => true + case (_: DataType.IntData, DataType.NullData) => true case _ => false } @@ -199,11 +238,14 @@ object DataType { } def allNumericTypes(l: List[DataType]): Boolean = - l.forall(d => d.isDouble || d.isInt) + l.forall(d => d.isDouble || d.isInt || d.isNull) + + def hasOptionalValues(l: List[DataType]): Boolean = + l.contains(DataType.NullData) def convertToBestType(l: List[DataType]): List[DataType] = // Cases we can manage: - // - They're all the same! + // - They're all the same! Maybe optional... // - Doubles and Ints, convert Ints to Doubles // - Fallback is that everything is a string. if (allSameType(l)) { @@ -213,11 +255,15 @@ object DataType { l.map { case v @ DataType.DoubleData(_) => v case v @ DataType.IntData(_) => v.toDoubleData + case DataType.NullData => DataType.NullData case s => throw new Exception(s"Unexpected non-numeric type '$s'") // Shouldn't get here. } } else { - // Nothing else to do, but make everything a string - l.map(_.toStringData) + // Nothing else to do, but make everything a string that isn't null. + l.map { + case DataType.NullData => DataType.NullData + case d => d.toStringData + } } } @@ -309,7 +355,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { object DataFrame { private val standardMessage: String = - "Embedded data must have two rows (minimum) of the same length (two columns minimum). The first row is the headers / field names. The first column are the keys. Cells cannot be empty." + "Embedded data must have two rows (minimum) of the same length (two columns minimum). The first row is the headers / field names. The first column are the keys." def fromRows(rows: List[List[DataType]]): DataFrame = rows match { diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala index 878eefd18..9f13db61c 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala @@ -5,13 +5,10 @@ class EmbedDataTests extends munit.FunSuite { test("Create a DataFrame") { val rows = - EmbedData.extractRows( - List( - "name,game,highscore,allowed", - "bob,tron,10000.00,true", - "Fred,tanks,476,false" - ), - "," + List( + EmbedData.extractRowData("name,game,highscore,allowed", ","), + EmbedData.extractRowData("bob,tron,10000.00,true", ","), + EmbedData.extractRowData("Fred,tanks,476,false", ",") ) val actual = @@ -87,7 +84,7 @@ class EmbedDataTests extends munit.FunSuite { } test("Extract row data - csv - simple") { - val row = " abc,123, def,456.5 ,ghi789,true ,." + val row = " abc,123, def,456.5 ,ghi789,true ,.,null, ," val actual = EmbedData.extractRowData(row, ",") @@ -100,7 +97,9 @@ class EmbedDataTests extends munit.FunSuite { DataType.DoubleData(456.5), DataType.StringData("ghi789"), DataType.BooleanData(true), - DataType.StringData(".") + DataType.StringData("."), + DataType.NullData, + DataType.NullData ) assertEquals(actual, expected) From 656f18d60e4120a7bf0244f2740739427be3fa35 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Sun, 19 Nov 2023 08:29:53 +0000 Subject: [PATCH 2/5] Empty cells are 'null', "null" is a string --- .../indigo-plugin/src/indigoplugin/generators/EmbedData.scala | 2 +- .../test/src/indigoplugin/generators/EmbedDataTests.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala index 523504100..2fea2b1f0 100644 --- a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala +++ b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala @@ -204,7 +204,7 @@ object DataType { private val isBoolean: Regex = """^(true|false)$""".r private val isDouble: Regex = """^([0-9]+?)\.([0-9]+)$""".r private val isInt: Regex = """^([0-9]+)$""".r - private val isNull: Regex = """^(null)$""".r + private val isNull: Regex = """^$""".r def decideType: String => DataType = { case isBoolean(v) => BooleanData(v.toBoolean) diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala index 9f13db61c..254373936 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala @@ -84,7 +84,7 @@ class EmbedDataTests extends munit.FunSuite { } test("Extract row data - csv - simple") { - val row = " abc,123, def,456.5 ,ghi789,true ,.,null, ," + val row = " abc,123, def,456.5 ,ghi789,true ,.,, ," val actual = EmbedData.extractRowData(row, ",") From 0fca5df77630e448cd53bcf9eed193dfa469f77a Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Sun, 19 Nov 2023 09:11:40 +0000 Subject: [PATCH 3/5] Handle negative ints and doubles --- .../indigoplugin/generators/EmbedData.scala | 17 +++++++----- .../generators/EmbedDataTests.scala | 27 +++++++++++++++++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala index 2fea2b1f0..ee3b73a86 100644 --- a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala +++ b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala @@ -202,8 +202,8 @@ object DataType { case object NullData extends DataType private val isBoolean: Regex = """^(true|false)$""".r - private val isDouble: Regex = """^([0-9]+?)\.([0-9]+)$""".r - private val isInt: Regex = """^([0-9]+)$""".r + private val isInt: Regex = """^(\-?[0-9]+)$""".r + private val isDouble: Regex = """^(\-?[0-9]+?)\.([0-9]+)$""".r private val isNull: Regex = """^$""".r def decideType: String => DataType = { @@ -276,12 +276,17 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { data.tail def alignColumnTypes: DataFrame = { - val transposed = rows.transpose - val stringKeys: Array[DataType] = transposed.head.map(_.toStringData) - val typeRows: Array[Array[DataType]] = transposed.tail + val columns = + rows.transpose + + val stringKeys: Array[DataType] = + columns.head.map(_.toStringData) + + val typedColumns: Array[Array[DataType]] = columns.tail .map(d => DataType.convertToBestType(d.toList).toArray) + val cleanedRows: Array[Array[DataType]] = - (stringKeys +: typeRows).transpose + (stringKeys +: typedColumns).transpose this.copy( data = headers.asInstanceOf[Array[DataType]] +: cleanedRows diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala index 254373936..6dbd46ab3 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala @@ -8,7 +8,8 @@ class EmbedDataTests extends munit.FunSuite { List( EmbedData.extractRowData("name,game,highscore,allowed", ","), EmbedData.extractRowData("bob,tron,10000.00,true", ","), - EmbedData.extractRowData("Fred,tanks,476,false", ",") + EmbedData.extractRowData("Fred,tanks,476,false", ","), + EmbedData.extractRowData("Stan,,-2,true", ",") ) val actual = @@ -35,6 +36,12 @@ class EmbedDataTests extends munit.FunSuite { DataType.StringData("tanks"), DataType.DoubleData(476.0), DataType.BooleanData(false) + ), + List( + DataType.StringData("Stan"), + DataType.NullData, + DataType.DoubleData(-2), + DataType.BooleanData(true) ) ) @@ -49,6 +56,7 @@ class EmbedDataTests extends munit.FunSuite { |enum GameScores(val game: String, val highscore: Double, val allowed: Boolean): | case Bob extends GameScores("tron", 10000.0, true) | case Fred extends GameScores("tanks", 476.0, false) + | case Stan extends GameScores(null, -2.0, true) """.stripMargin assertEquals(actualEnum.trim, expectedEnum.trim) @@ -61,6 +69,7 @@ class EmbedDataTests extends munit.FunSuite { |enum GameScores(val game: String, val highscore: Double, val allowed: Boolean) extends ScoreData: | case Bob extends GameScores("tron", 10000.0, true) | case Fred extends GameScores("tanks", 476.0, false) + | case Stan extends GameScores(null, -2.0, true) """.stripMargin assertEquals(actualEnumWithExtends.trim, expectedEnumWithExtends.trim) @@ -75,7 +84,8 @@ class EmbedDataTests extends munit.FunSuite { | val data: Map[String, GameScores] = | Map( | "bob" -> GameScores("tron", 10000.0, true), - | "Fred" -> GameScores("tanks", 476.0, false) + | "Fred" -> GameScores("tanks", 476.0, false), + | "Stan" -> GameScores(null, -2.0, true) | ) """.stripMargin @@ -175,4 +185,17 @@ class EmbedDataTests extends munit.FunSuite { assertEquals(actual, expected) } + test("decideType - int") { + assertEquals(DataType.decideType("10"), DataType.IntData(10)) + assertEquals(DataType.decideType("-10"), DataType.IntData(-10)) + } + + test("decideType - double") { + assertEquals(DataType.decideType("10.0"), DataType.DoubleData(10.0)) + assertEquals(DataType.decideType("-10.0"), DataType.DoubleData(-10.0)) + assertEquals(DataType.decideType("-10."), DataType.StringData("-10.")) + assertEquals(DataType.decideType(".0"), DataType.StringData(".0")) + assertEquals(DataType.decideType("."), DataType.StringData(".")) + } + } From 459a77524f44f842ebd6e06f51017a4e92ce37d8 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Sun, 19 Nov 2023 10:23:59 +0000 Subject: [PATCH 4/5] Acceptance tests failing --- .../indigoplugin/generators/EmbedData.scala | 133 +++++++++++++----- .../generators/EmbedDataTests.scala | 54 +++---- .../generators/GeneratorAcceptanceTests.scala | 2 +- indigo-plugin/test-assets/data/stats.csv | 8 +- 4 files changed, 126 insertions(+), 71 deletions(-) diff --git a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala index ee3b73a86..18428fa29 100644 --- a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala +++ b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala @@ -95,12 +95,12 @@ object EmbedData { } parse(delimiter)(cleanRow).map(_._1).collect { - case d @ DataType.StringData(s) if s.nonEmpty => d - case DataType.StringData(_) => DataType.NullData - case d: DataType.BooleanData => d - case d: DataType.DoubleData => d - case d: DataType.IntData => d - case DataType.NullData => DataType.NullData + case d @ DataType.StringData(s, _) if s.nonEmpty => d + case DataType.StringData(_, _) => DataType.NullData + case d: DataType.BooleanData => d + case d: DataType.DoubleData => d + case d: DataType.IntData => d + case DataType.NullData => DataType.NullData } } @@ -132,6 +132,9 @@ object EmbedData { sealed trait DataType { + def nullable: Boolean + def makeOptional: DataType + def isString: Boolean = this match { case _: DataType.StringData => true @@ -164,42 +167,80 @@ sealed trait DataType { def toStringData: DataType.StringData = this match { - case s: DataType.StringData => s - case DataType.BooleanData(value) => DataType.StringData(value.toString) - case DataType.DoubleData(value) => DataType.StringData(value.toString) - case DataType.IntData(value) => DataType.StringData(value.toString) - case DataType.NullData => DataType.StringData("null") + case s: DataType.StringData if s.nullable => DataType.StringData(s"""Some("${s.value}")""", true) + case s: DataType.StringData => s + case DataType.BooleanData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true) + case DataType.BooleanData(value, _) => DataType.StringData(value.toString, false) + case DataType.DoubleData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true) + case DataType.DoubleData(value, _) => DataType.StringData(value.toString, false) + case DataType.IntData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true) + case DataType.IntData(value, _) => DataType.StringData(value.toString, false) + case DataType.NullData => DataType.StringData("None", true) } def asString: String = this match { - case s: DataType.StringData => s""""${s.value}"""" - case DataType.BooleanData(value) => value.toString - case DataType.DoubleData(value) => value.toString - case DataType.IntData(value) => value.toString - case DataType.NullData => "null" + case s: DataType.StringData if s.nullable => s"""Some("${s.value}")""" + case s: DataType.StringData => s""""${s.value}"""" + case DataType.BooleanData(value, nullable) if nullable => s"Some(${value.toString})" + case DataType.BooleanData(value, _) => value.toString + case DataType.DoubleData(value, nullable) if nullable => s"Some(${value.toString})" + case DataType.DoubleData(value, _) => value.toString + case DataType.IntData(value, nullable) if nullable => s"Some(${value.toString})" + case DataType.IntData(value, _) => value.toString + case DataType.NullData => "None" } def giveTypeName: String = this match { - case _: DataType.StringData => "String" - case _: DataType.BooleanData => "Boolean" - case _: DataType.DoubleData => "Double" - case _: DataType.IntData => "Int" - case DataType.NullData => "Null" + case d: DataType.StringData if d.nullable => "Option[String]" + case _: DataType.StringData => "String" + case d: DataType.BooleanData if d.nullable => "Option[Boolean]" + case _: DataType.BooleanData => "Boolean" + case d: DataType.DoubleData if d.nullable => "Option[Double]" + case _: DataType.DoubleData => "Double" + case d: DataType.IntData if d.nullable => "Option[Int]" + case _: DataType.IntData => "Int" + case DataType.NullData => "Null" } } object DataType { // Most to least specific: Boolean, Int, Double, String - final case class BooleanData(value: Boolean) extends DataType - final case class IntData(value: Int) extends DataType { - def toDoubleData: DoubleData = DoubleData(value.toDouble) + final case class BooleanData(value: Boolean, nullable: Boolean) extends DataType { + def makeOptional: BooleanData = this.copy(nullable = true) + } + object BooleanData { + def apply(value: Boolean): BooleanData = BooleanData(value, false) + } + + final case class IntData(value: Int, nullable: Boolean) extends DataType { + def toDoubleData: DoubleData = DoubleData(value.toDouble, nullable) + def makeOptional: IntData = this.copy(nullable = true) + } + object IntData { + def apply(value: Int): IntData = IntData(value, false) + } + + final case class DoubleData(value: Double, nullable: Boolean) extends DataType { + def makeOptional: DoubleData = this.copy(nullable = true) + } + object DoubleData { + def apply(value: Double): DoubleData = DoubleData(value, false) + } + + final case class StringData(value: String, nullable: Boolean) extends DataType { + def makeOptional: StringData = this.copy(nullable = true) + } + object StringData { + def apply(value: String): StringData = StringData(value, false) + } + + case object NullData extends DataType { + val nullable: Boolean = true + def makeOptional: DataType = this } - final case class DoubleData(value: Double) extends DataType - final case class StringData(value: String) extends DataType - case object NullData extends DataType private val isBoolean: Regex = """^(true|false)$""".r private val isInt: Regex = """^(\-?[0-9]+)$""".r @@ -207,11 +248,11 @@ object DataType { private val isNull: Regex = """^$""".r def decideType: String => DataType = { - case isBoolean(v) => BooleanData(v.toBoolean) - case isInt(v) => IntData(v.toInt) - case isDouble(v1, v2) => DoubleData(s"$v1.$v2".toDouble) + case isBoolean(v) => BooleanData(v.toBoolean, false) + case isInt(v) => IntData(v.toInt, false) + case isDouble(v1, v2) => DoubleData(s"$v1.$v2".toDouble, false) case isNull(_) => NullData - case v => StringData(v) + case v => StringData(v, false) } def sameType(a: DataType, b: DataType): Boolean = @@ -253,9 +294,9 @@ object DataType { l } else if (allNumericTypes(l)) { l.map { - case v @ DataType.DoubleData(_) => v - case v @ DataType.IntData(_) => v.toDoubleData - case DataType.NullData => DataType.NullData + case v @ DataType.DoubleData(_, _) => v + case v @ DataType.IntData(_, _) => v.toDoubleData + case DataType.NullData => DataType.NullData case s => throw new Exception(s"Unexpected non-numeric type '$s'") // Shouldn't get here. } } else { @@ -285,8 +326,17 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { val typedColumns: Array[Array[DataType]] = columns.tail .map(d => DataType.convertToBestType(d.toList).toArray) + val optionalColumns: Array[Array[DataType]] = + typedColumns.map { col => + if (DataType.hasOptionalValues(col.toList)) { + col.map(_.makeOptional) + } else { + col + } + } + val cleanedRows: Array[Array[DataType]] = - (stringKeys +: typedColumns).transpose + (stringKeys +: optionalColumns).transpose this.copy( data = headers.asInstanceOf[Array[DataType]] +: cleanedRows @@ -308,10 +358,15 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { } } - def renderVars: String = { + def renderVars(omitVal: Boolean): String = { val names = headers.drop(1).map(_.value) val types = rows.head.drop(1).map(_.giveTypeName) - names.zip(types).map { case (n, t) => s"val ${toSafeNameCamel(n)}: $t" }.mkString(", ") + names + .zip(types) + .map { case (n, t) => + (if (omitVal) "" else "val ") + s"${toSafeNameCamel(n)}: $t" + } + .mkString(", ") } def renderEnum(moduleName: String, extendsFrom: Option[String]): String = { @@ -331,7 +386,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { .getOrElse("") s""" - |enum $moduleName(${renderVars})$extFrom: + |enum $moduleName(${renderVars(false)})$extFrom: |${renderedRows} |""".stripMargin } @@ -345,7 +400,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { .mkString(",\n") s""" - |final case class $moduleName(${renderVars}) + |final case class $moduleName(${renderVars(true)}) |object $moduleName: | val data: Map[String, $moduleName] = | Map( diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala index 6dbd46ab3..b42d62e25 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala @@ -17,31 +17,31 @@ class EmbedDataTests extends munit.FunSuite { val expectedHeaders = List( - DataType.StringData("name"), - DataType.StringData("game"), - DataType.StringData("highscore"), - DataType.StringData("allowed") + DataType.StringData("name", false), + DataType.StringData("game", false), + DataType.StringData("highscore", false), + DataType.StringData("allowed", false) ) val expectedRows = List( List( - DataType.StringData("bob"), - DataType.StringData("tron"), - DataType.DoubleData(10000.0), - DataType.BooleanData(true) + DataType.StringData("bob", false), + DataType.StringData("tron", true), + DataType.DoubleData(10000.0, false), + DataType.BooleanData(true, false) ), List( - DataType.StringData("Fred"), - DataType.StringData("tanks"), - DataType.DoubleData(476.0), - DataType.BooleanData(false) + DataType.StringData("Fred", false), + DataType.StringData("tanks", true), + DataType.DoubleData(476.0, false), + DataType.BooleanData(false, false) ), List( - DataType.StringData("Stan"), + DataType.StringData("Stan", false), DataType.NullData, - DataType.DoubleData(-2), - DataType.BooleanData(true) + DataType.DoubleData(-2, false), + DataType.BooleanData(true, false) ) ) @@ -53,10 +53,10 @@ class EmbedDataTests extends munit.FunSuite { val expectedEnum = """ - |enum GameScores(val game: String, val highscore: Double, val allowed: Boolean): - | case Bob extends GameScores("tron", 10000.0, true) - | case Fred extends GameScores("tanks", 476.0, false) - | case Stan extends GameScores(null, -2.0, true) + |enum GameScores(val game: Option[String], val highscore: Double, val allowed: Boolean): + | case Bob extends GameScores(Some("tron"), 10000.0, true) + | case Fred extends GameScores(Some("tanks"), 476.0, false) + | case Stan extends GameScores(None, -2.0, true) """.stripMargin assertEquals(actualEnum.trim, expectedEnum.trim) @@ -66,10 +66,10 @@ class EmbedDataTests extends munit.FunSuite { val expectedEnumWithExtends = """ - |enum GameScores(val game: String, val highscore: Double, val allowed: Boolean) extends ScoreData: - | case Bob extends GameScores("tron", 10000.0, true) - | case Fred extends GameScores("tanks", 476.0, false) - | case Stan extends GameScores(null, -2.0, true) + |enum GameScores(val game: Option[String], val highscore: Double, val allowed: Boolean) extends ScoreData: + | case Bob extends GameScores(Some("tron"), 10000.0, true) + | case Fred extends GameScores(Some("tanks"), 476.0, false) + | case Stan extends GameScores(None, -2.0, true) """.stripMargin assertEquals(actualEnumWithExtends.trim, expectedEnumWithExtends.trim) @@ -79,13 +79,13 @@ class EmbedDataTests extends munit.FunSuite { val expectedMap = """ - |final case class GameScores(val game: String, val highscore: Double, val allowed: Boolean) + |final case class GameScores(game: Option[String], highscore: Double, allowed: Boolean) |object GameScores: | val data: Map[String, GameScores] = | Map( - | "bob" -> GameScores("tron", 10000.0, true), - | "Fred" -> GameScores("tanks", 476.0, false), - | "Stan" -> GameScores(null, -2.0, true) + | "bob" -> GameScores(Some("tron"), 10000.0, true), + | "Fred" -> GameScores(Some("tanks"), 476.0, false), + | "Stan" -> GameScores(None, -2.0, true) | ) """.stripMargin diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala index 7ca94ec2d..709bc2f8d 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala @@ -72,7 +72,7 @@ class GeneratorAcceptanceTests extends munit.FunSuite { | |// DO NOT EDIT: Generated by Indigo. | - |final case class StatsMap(val level: Int, val bonus: Int, val code: String) + |final case class StatsMap(level: Int, bonus: Int, code: String) |object StatsMap: | val data: Map[String, StatsMap] = | Map( diff --git a/indigo-plugin/test-assets/data/stats.csv b/indigo-plugin/test-assets/data/stats.csv index e0b053e51..8f5d6dcba 100644 --- a/indigo-plugin/test-assets/data/stats.csv +++ b/indigo-plugin/test-assets/data/stats.csv @@ -1,5 +1,5 @@ -name,level,bonus -intelligence,2,4 -strength,10,0 -fortitude,4,1 +name,level,bonus,stackable +intelligence,2,4,true +strength,10,0, +fortitude,4,1,false From 5273640f22fe4e95c230f8f6998b6dc11d408554 Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Sun, 19 Nov 2023 16:22:18 +0000 Subject: [PATCH 5/5] Tests pass with optional fields --- .../indigoplugin/generators/EmbedData.scala | 35 +++++++++++---- .../generators/EmbedDataTests.scala | 45 +++++++++++++++++++ .../generators/GeneratorAcceptanceTests.scala | 22 ++++----- indigo-plugin/test-assets/data/stats.md | 2 +- 4 files changed, 84 insertions(+), 20 deletions(-) diff --git a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala index 18428fa29..1ff3e29dc 100644 --- a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala +++ b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala @@ -33,10 +33,11 @@ object EmbedData { os.read.lines(filePath).filter(rowFilter) } + val rows = + lines.map(row => extractRowData(row, delimiter)).toList + val dataFrame = - DataFrame.fromRows( - lines.map(row => extractRowData(row, delimiter)).toList - ) + DataFrame.fromRows(rows) val wd = outDir / Generators.OutputDirName @@ -201,7 +202,7 @@ sealed trait DataType { case _: DataType.DoubleData => "Double" case d: DataType.IntData if d.nullable => "Option[Int]" case _: DataType.IntData => "Int" - case DataType.NullData => "Null" + case DataType.NullData => "Option[Any]" } } @@ -307,6 +308,26 @@ object DataType { } } + def matchHeaderRowLength(rows: Array[Array[DataType]]): Array[Array[DataType]] = + rows.toList match { + case Nil => + rows + + case headers :: data => + val l = headers.length + val res = + headers :: data.map { r => + val diff = l - r.length + if (diff > 0) { + r ++ List.fill(diff)(DataType.NullData) + } else { + r + } + } + + res.toArray + } + } final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { @@ -318,7 +339,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { def alignColumnTypes: DataFrame = { val columns = - rows.transpose + DataType.matchHeaderRowLength(rows).transpose val stringKeys: Array[DataType] = columns.head.map(_.toStringData) @@ -425,15 +446,13 @@ object DataFrame { case _ :: Nil => throw new Exception("Only one row of data found. " + standardMessage) - case h :: t => + case h :: _ => val len = h.length if (len == 0) { throw new Exception("No data to create. " + standardMessage) } else if (len == 1) { throw new Exception("Only one column of data. " + standardMessage) - } else if (!t.forall(_.length == len)) { - throw new Exception(s"All rows must be the same length. Header row had '$len' columns. " + standardMessage) } else { DataFrame(rows.map(_.toArray).toArray, len).alignColumnTypes } diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala index b42d62e25..b64e75690 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala @@ -198,4 +198,49 @@ class EmbedDataTests extends munit.FunSuite { assertEquals(DataType.decideType("."), DataType.StringData(".")) } + test("matchHeaderRowLength") { + + val rows = + List( + EmbedData.extractRowData("name,game,highscore,allowed", ","), + EmbedData.extractRowData("bob", ","), + EmbedData.extractRowData("Fred,tanks,476,false", ","), + EmbedData.extractRowData("Stan,,-2", ",") + ) + + val actual = + DataType.matchHeaderRowLength(rows.map(_.toArray).toArray).map(_.toList).toList + + val expected = + List( + List( + DataType.StringData("name", false), + DataType.StringData("game", false), + DataType.StringData("highscore", false), + DataType.StringData("allowed", false) + ), + List( + DataType.StringData("bob", false), + DataType.NullData, + DataType.NullData, + DataType.NullData + ), + List( + DataType.StringData("Fred", false), + DataType.StringData("tanks", false), + DataType.IntData(476, false), + DataType.BooleanData(false, false) + ), + List( + DataType.StringData("Stan", false), + DataType.NullData, + DataType.IntData(-2, false), + DataType.NullData + ) + ) + + assertEquals(actual, expected) + + } + } diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala index 709bc2f8d..8da4ee1fe 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala @@ -40,10 +40,10 @@ class GeneratorAcceptanceTests extends munit.FunSuite { | |// DO NOT EDIT: Generated by Indigo. | - |enum StatsEnum(val level: Int, val bonus: Int): - | case Intelligence extends StatsEnum(2, 4) - | case Strength extends StatsEnum(10, 0) - | case Fortitude extends StatsEnum(4, 1) + |enum StatsEnum(val level: Int, val bonus: Int, val stackable: Option[Boolean]): + | case Intelligence extends StatsEnum(2, 4, Some(true)) + | case Strength extends StatsEnum(10, 0, None) + | case Fortitude extends StatsEnum(4, 1, Some(false)) |""".stripMargin assertEquals(actual.trim, expected.trim) @@ -72,13 +72,13 @@ class GeneratorAcceptanceTests extends munit.FunSuite { | |// DO NOT EDIT: Generated by Indigo. | - |final case class StatsMap(level: Int, bonus: Int, code: String) + |final case class StatsMap(level: Int, bonus: Int, code: Option[String]) |object StatsMap: | val data: Map[String, StatsMap] = | Map( - | "intelligence" -> StatsMap(2, 4, "i"), - | "strength" -> StatsMap(10, 0, "st"), - | "fortitude" -> StatsMap(4, 1, "_frt") + | "intelligence" -> StatsMap(2, 4, Some("i")), + | "strength" -> StatsMap(10, 0, None), + | "fortitude" -> StatsMap(4, 1, Some("_frt")) | ) |""".stripMargin @@ -144,9 +144,9 @@ class GeneratorAcceptanceTests extends munit.FunSuite { |// DO NOT EDIT: Generated by Indigo. |/* |"name": String,"level": String,"bonus": String,"code": String - |"intelligence": String,2: Int,4: Int,"i": String - |"strength": String,10: Int,0: Int,"st": String - |"fortitude": String,4: Int,1: Int,"_frt": String + |"intelligence": String,2: Int,4: Int,Some("i"): Option[String] + |"strength": String,10: Int,0: Int,None: Option[Any] + |"fortitude": String,4: Int,1: Int,Some("_frt"): Option[String] |*/ |""".stripMargin diff --git a/indigo-plugin/test-assets/data/stats.md b/indigo-plugin/test-assets/data/stats.md index 8674df389..5dacb745f 100644 --- a/indigo-plugin/test-assets/data/stats.md +++ b/indigo-plugin/test-assets/data/stats.md @@ -1,5 +1,5 @@ | name | level | bonus | code | | ------------ | ----- | ----- | ---- | | intelligence | 2 | 4 | i | -| strength | 10 | 0 | st | +| strength | 10 | 0 | | | fortitude | 4 | 1 | _frt |