diff --git a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala index c7483e08e..1ff3e29dc 100644 --- a/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala +++ b/indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala @@ -33,10 +33,11 @@ object EmbedData { os.read.lines(filePath).filter(rowFilter) } + val rows = + lines.map(row => extractRowData(row, delimiter)).toList + val dataFrame = - DataFrame.fromRows( - lines.map(row => extractRowData(row, delimiter)).toList - ) + DataFrame.fromRows(rows) val wd = outDir / Generators.OutputDirName @@ -74,16 +75,35 @@ object EmbedData { Seq(file) } - def extractRows(rows: List[String], delimiter: String): List[List[DataType]] = - rows.map(r => extractRowData(r, delimiter)) + def extractRowData(row: String, delimiter: String): List[DataType] = { + + val cleanDelimiter: String = + if (delimiter == "\\|") "|" else delimiter + + val cleanRow: String = + row.trim match { + case r if r.startsWith(cleanDelimiter) && r.endsWith(cleanDelimiter) => + r.drop(cleanDelimiter.length()).dropRight(cleanDelimiter.length()) + + case r if r.startsWith(cleanDelimiter) => + r.drop(cleanDelimiter.length()) + + case r if r.endsWith(cleanDelimiter) => + r.dropRight(cleanDelimiter.length()) - def extractRowData(row: String, delimiter: String): List[DataType] = - parse(delimiter)(row).map(_._1).collect { - case d @ DataType.StringData(s) if s.nonEmpty => d - case d: DataType.BooleanData => d - case d: DataType.DoubleData => d - case d: DataType.IntData => d + case r => + r + } + + parse(delimiter)(cleanRow).map(_._1).collect { + case d @ DataType.StringData(s, _) if s.nonEmpty => d + case DataType.StringData(_, _) => DataType.NullData + case d: DataType.BooleanData => d + case d: DataType.DoubleData => d + case d: DataType.IntData => d + case DataType.NullData => DataType.NullData } + } // A parser of things, // is a function from strings, @@ -113,6 +133,9 @@ object EmbedData { sealed trait DataType { + def nullable: Boolean + def makeOptional: DataType + def isString: Boolean = this match { case _: DataType.StringData => true @@ -137,58 +160,116 @@ sealed trait DataType { case _ => false } + def isNull: Boolean = + this match { + case DataType.NullData => true + case _ => false + } + def toStringData: DataType.StringData = this match { - case s: DataType.StringData => s - case DataType.BooleanData(value) => DataType.StringData(value.toString) - case DataType.DoubleData(value) => DataType.StringData(value.toString) - case DataType.IntData(value) => DataType.StringData(value.toString) + case s: DataType.StringData if s.nullable => DataType.StringData(s"""Some("${s.value}")""", true) + case s: DataType.StringData => s + case DataType.BooleanData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true) + case DataType.BooleanData(value, _) => DataType.StringData(value.toString, false) + case DataType.DoubleData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true) + case DataType.DoubleData(value, _) => DataType.StringData(value.toString, false) + case DataType.IntData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true) + case DataType.IntData(value, _) => DataType.StringData(value.toString, false) + case DataType.NullData => DataType.StringData("None", true) } def asString: String = this match { - case s: DataType.StringData => s""""${s.value}"""" - case DataType.BooleanData(value) => value.toString - case DataType.DoubleData(value) => value.toString - case DataType.IntData(value) => value.toString + case s: DataType.StringData if s.nullable => s"""Some("${s.value}")""" + case s: DataType.StringData => s""""${s.value}"""" + case DataType.BooleanData(value, nullable) if nullable => s"Some(${value.toString})" + case DataType.BooleanData(value, _) => value.toString + case DataType.DoubleData(value, nullable) if nullable => s"Some(${value.toString})" + case DataType.DoubleData(value, _) => value.toString + case DataType.IntData(value, nullable) if nullable => s"Some(${value.toString})" + case DataType.IntData(value, _) => value.toString + case DataType.NullData => "None" } def giveTypeName: String = this match { - case _: DataType.StringData => "String" - case _: DataType.BooleanData => "Boolean" - case _: DataType.DoubleData => "Double" - case _: DataType.IntData => "Int" + case d: DataType.StringData if d.nullable => "Option[String]" + case _: DataType.StringData => "String" + case d: DataType.BooleanData if d.nullable => "Option[Boolean]" + case _: DataType.BooleanData => "Boolean" + case d: DataType.DoubleData if d.nullable => "Option[Double]" + case _: DataType.DoubleData => "Double" + case d: DataType.IntData if d.nullable => "Option[Int]" + case _: DataType.IntData => "Int" + case DataType.NullData => "Option[Any]" } } object DataType { // Most to least specific: Boolean, Int, Double, String - final case class BooleanData(value: Boolean) extends DataType - final case class IntData(value: Int) extends DataType { - def toDoubleData: DoubleData = DoubleData(value.toDouble) + final case class BooleanData(value: Boolean, nullable: Boolean) extends DataType { + def makeOptional: BooleanData = this.copy(nullable = true) + } + object BooleanData { + def apply(value: Boolean): BooleanData = BooleanData(value, false) + } + + final case class IntData(value: Int, nullable: Boolean) extends DataType { + def toDoubleData: DoubleData = DoubleData(value.toDouble, nullable) + def makeOptional: IntData = this.copy(nullable = true) + } + object IntData { + def apply(value: Int): IntData = IntData(value, false) + } + + final case class DoubleData(value: Double, nullable: Boolean) extends DataType { + def makeOptional: DoubleData = this.copy(nullable = true) + } + object DoubleData { + def apply(value: Double): DoubleData = DoubleData(value, false) + } + + final case class StringData(value: String, nullable: Boolean) extends DataType { + def makeOptional: StringData = this.copy(nullable = true) + } + object StringData { + def apply(value: String): StringData = StringData(value, false) + } + + case object NullData extends DataType { + val nullable: Boolean = true + def makeOptional: DataType = this } - final case class DoubleData(value: Double) extends DataType - final case class StringData(value: String) extends DataType private val isBoolean: Regex = """^(true|false)$""".r - private val isDouble: Regex = """^([0-9]+?)\.([0-9]+)$""".r - private val isInt: Regex = """^([0-9]+)$""".r + private val isInt: Regex = """^(\-?[0-9]+)$""".r + private val isDouble: Regex = """^(\-?[0-9]+?)\.([0-9]+)$""".r + private val isNull: Regex = """^$""".r def decideType: String => DataType = { - case isBoolean(v) => BooleanData(v.toBoolean) - case isInt(v) => IntData(v.toInt) - case isDouble(v1, v2) => DoubleData(s"$v1.$v2".toDouble) - case v => StringData(v) + case isBoolean(v) => BooleanData(v.toBoolean, false) + case isInt(v) => IntData(v.toInt, false) + case isDouble(v1, v2) => DoubleData(s"$v1.$v2".toDouble, false) + case isNull(_) => NullData + case v => StringData(v, false) } def sameType(a: DataType, b: DataType): Boolean = (a, b) match { case (_: DataType.StringData, _: DataType.StringData) => true + case (DataType.NullData, _: DataType.StringData) => true + case (_: DataType.StringData, DataType.NullData) => true case (_: DataType.BooleanData, _: DataType.BooleanData) => true + case (DataType.NullData, _: DataType.BooleanData) => true + case (_: DataType.BooleanData, DataType.NullData) => true case (_: DataType.DoubleData, _: DataType.DoubleData) => true + case (DataType.NullData, _: DataType.DoubleData) => true + case (_: DataType.DoubleData, DataType.NullData) => true case (_: DataType.IntData, _: DataType.IntData) => true + case (DataType.NullData, _: DataType.IntData) => true + case (_: DataType.IntData, DataType.NullData) => true case _ => false } @@ -199,11 +280,14 @@ object DataType { } def allNumericTypes(l: List[DataType]): Boolean = - l.forall(d => d.isDouble || d.isInt) + l.forall(d => d.isDouble || d.isInt || d.isNull) + + def hasOptionalValues(l: List[DataType]): Boolean = + l.contains(DataType.NullData) def convertToBestType(l: List[DataType]): List[DataType] = // Cases we can manage: - // - They're all the same! + // - They're all the same! Maybe optional... // - Doubles and Ints, convert Ints to Doubles // - Fallback is that everything is a string. if (allSameType(l)) { @@ -211,13 +295,37 @@ object DataType { l } else if (allNumericTypes(l)) { l.map { - case v @ DataType.DoubleData(_) => v - case v @ DataType.IntData(_) => v.toDoubleData + case v @ DataType.DoubleData(_, _) => v + case v @ DataType.IntData(_, _) => v.toDoubleData + case DataType.NullData => DataType.NullData case s => throw new Exception(s"Unexpected non-numeric type '$s'") // Shouldn't get here. } } else { - // Nothing else to do, but make everything a string - l.map(_.toStringData) + // Nothing else to do, but make everything a string that isn't null. + l.map { + case DataType.NullData => DataType.NullData + case d => d.toStringData + } + } + + def matchHeaderRowLength(rows: Array[Array[DataType]]): Array[Array[DataType]] = + rows.toList match { + case Nil => + rows + + case headers :: data => + val l = headers.length + val res = + headers :: data.map { r => + val diff = l - r.length + if (diff > 0) { + r ++ List.fill(diff)(DataType.NullData) + } else { + r + } + } + + res.toArray } } @@ -230,12 +338,26 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { data.tail def alignColumnTypes: DataFrame = { - val transposed = rows.transpose - val stringKeys: Array[DataType] = transposed.head.map(_.toStringData) - val typeRows: Array[Array[DataType]] = transposed.tail + val columns = + DataType.matchHeaderRowLength(rows).transpose + + val stringKeys: Array[DataType] = + columns.head.map(_.toStringData) + + val typedColumns: Array[Array[DataType]] = columns.tail .map(d => DataType.convertToBestType(d.toList).toArray) + + val optionalColumns: Array[Array[DataType]] = + typedColumns.map { col => + if (DataType.hasOptionalValues(col.toList)) { + col.map(_.makeOptional) + } else { + col + } + } + val cleanedRows: Array[Array[DataType]] = - (stringKeys +: typeRows).transpose + (stringKeys +: optionalColumns).transpose this.copy( data = headers.asInstanceOf[Array[DataType]] +: cleanedRows @@ -257,10 +379,15 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { } } - def renderVars: String = { + def renderVars(omitVal: Boolean): String = { val names = headers.drop(1).map(_.value) val types = rows.head.drop(1).map(_.giveTypeName) - names.zip(types).map { case (n, t) => s"val ${toSafeNameCamel(n)}: $t" }.mkString(", ") + names + .zip(types) + .map { case (n, t) => + (if (omitVal) "" else "val ") + s"${toSafeNameCamel(n)}: $t" + } + .mkString(", ") } def renderEnum(moduleName: String, extendsFrom: Option[String]): String = { @@ -280,7 +407,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { .getOrElse("") s""" - |enum $moduleName(${renderVars})$extFrom: + |enum $moduleName(${renderVars(false)})$extFrom: |${renderedRows} |""".stripMargin } @@ -294,7 +421,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { .mkString(",\n") s""" - |final case class $moduleName(${renderVars}) + |final case class $moduleName(${renderVars(true)}) |object $moduleName: | val data: Map[String, $moduleName] = | Map( @@ -309,7 +436,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) { object DataFrame { private val standardMessage: String = - "Embedded data must have two rows (minimum) of the same length (two columns minimum). The first row is the headers / field names. The first column are the keys. Cells cannot be empty." + "Embedded data must have two rows (minimum) of the same length (two columns minimum). The first row is the headers / field names. The first column are the keys." def fromRows(rows: List[List[DataType]]): DataFrame = rows match { @@ -319,15 +446,13 @@ object DataFrame { case _ :: Nil => throw new Exception("Only one row of data found. " + standardMessage) - case h :: t => + case h :: _ => val len = h.length if (len == 0) { throw new Exception("No data to create. " + standardMessage) } else if (len == 1) { throw new Exception("Only one column of data. " + standardMessage) - } else if (!t.forall(_.length == len)) { - throw new Exception(s"All rows must be the same length. Header row had '$len' columns. " + standardMessage) } else { DataFrame(rows.map(_.toArray).toArray, len).alignColumnTypes } diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala index 878eefd18..b64e75690 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/EmbedDataTests.scala @@ -5,13 +5,11 @@ class EmbedDataTests extends munit.FunSuite { test("Create a DataFrame") { val rows = - EmbedData.extractRows( - List( - "name,game,highscore,allowed", - "bob,tron,10000.00,true", - "Fred,tanks,476,false" - ), - "," + List( + EmbedData.extractRowData("name,game,highscore,allowed", ","), + EmbedData.extractRowData("bob,tron,10000.00,true", ","), + EmbedData.extractRowData("Fred,tanks,476,false", ","), + EmbedData.extractRowData("Stan,,-2,true", ",") ) val actual = @@ -19,25 +17,31 @@ class EmbedDataTests extends munit.FunSuite { val expectedHeaders = List( - DataType.StringData("name"), - DataType.StringData("game"), - DataType.StringData("highscore"), - DataType.StringData("allowed") + DataType.StringData("name", false), + DataType.StringData("game", false), + DataType.StringData("highscore", false), + DataType.StringData("allowed", false) ) val expectedRows = List( List( - DataType.StringData("bob"), - DataType.StringData("tron"), - DataType.DoubleData(10000.0), - DataType.BooleanData(true) + DataType.StringData("bob", false), + DataType.StringData("tron", true), + DataType.DoubleData(10000.0, false), + DataType.BooleanData(true, false) + ), + List( + DataType.StringData("Fred", false), + DataType.StringData("tanks", true), + DataType.DoubleData(476.0, false), + DataType.BooleanData(false, false) ), List( - DataType.StringData("Fred"), - DataType.StringData("tanks"), - DataType.DoubleData(476.0), - DataType.BooleanData(false) + DataType.StringData("Stan", false), + DataType.NullData, + DataType.DoubleData(-2, false), + DataType.BooleanData(true, false) ) ) @@ -49,9 +53,10 @@ class EmbedDataTests extends munit.FunSuite { val expectedEnum = """ - |enum GameScores(val game: String, val highscore: Double, val allowed: Boolean): - | case Bob extends GameScores("tron", 10000.0, true) - | case Fred extends GameScores("tanks", 476.0, false) + |enum GameScores(val game: Option[String], val highscore: Double, val allowed: Boolean): + | case Bob extends GameScores(Some("tron"), 10000.0, true) + | case Fred extends GameScores(Some("tanks"), 476.0, false) + | case Stan extends GameScores(None, -2.0, true) """.stripMargin assertEquals(actualEnum.trim, expectedEnum.trim) @@ -61,9 +66,10 @@ class EmbedDataTests extends munit.FunSuite { val expectedEnumWithExtends = """ - |enum GameScores(val game: String, val highscore: Double, val allowed: Boolean) extends ScoreData: - | case Bob extends GameScores("tron", 10000.0, true) - | case Fred extends GameScores("tanks", 476.0, false) + |enum GameScores(val game: Option[String], val highscore: Double, val allowed: Boolean) extends ScoreData: + | case Bob extends GameScores(Some("tron"), 10000.0, true) + | case Fred extends GameScores(Some("tanks"), 476.0, false) + | case Stan extends GameScores(None, -2.0, true) """.stripMargin assertEquals(actualEnumWithExtends.trim, expectedEnumWithExtends.trim) @@ -73,12 +79,13 @@ class EmbedDataTests extends munit.FunSuite { val expectedMap = """ - |final case class GameScores(val game: String, val highscore: Double, val allowed: Boolean) + |final case class GameScores(game: Option[String], highscore: Double, allowed: Boolean) |object GameScores: | val data: Map[String, GameScores] = | Map( - | "bob" -> GameScores("tron", 10000.0, true), - | "Fred" -> GameScores("tanks", 476.0, false) + | "bob" -> GameScores(Some("tron"), 10000.0, true), + | "Fred" -> GameScores(Some("tanks"), 476.0, false), + | "Stan" -> GameScores(None, -2.0, true) | ) """.stripMargin @@ -87,7 +94,7 @@ class EmbedDataTests extends munit.FunSuite { } test("Extract row data - csv - simple") { - val row = " abc,123, def,456.5 ,ghi789,true ,." + val row = " abc,123, def,456.5 ,ghi789,true ,.,, ," val actual = EmbedData.extractRowData(row, ",") @@ -100,7 +107,9 @@ class EmbedDataTests extends munit.FunSuite { DataType.DoubleData(456.5), DataType.StringData("ghi789"), DataType.BooleanData(true), - DataType.StringData(".") + DataType.StringData("."), + DataType.NullData, + DataType.NullData ) assertEquals(actual, expected) @@ -176,4 +185,62 @@ class EmbedDataTests extends munit.FunSuite { assertEquals(actual, expected) } + test("decideType - int") { + assertEquals(DataType.decideType("10"), DataType.IntData(10)) + assertEquals(DataType.decideType("-10"), DataType.IntData(-10)) + } + + test("decideType - double") { + assertEquals(DataType.decideType("10.0"), DataType.DoubleData(10.0)) + assertEquals(DataType.decideType("-10.0"), DataType.DoubleData(-10.0)) + assertEquals(DataType.decideType("-10."), DataType.StringData("-10.")) + assertEquals(DataType.decideType(".0"), DataType.StringData(".0")) + assertEquals(DataType.decideType("."), DataType.StringData(".")) + } + + test("matchHeaderRowLength") { + + val rows = + List( + EmbedData.extractRowData("name,game,highscore,allowed", ","), + EmbedData.extractRowData("bob", ","), + EmbedData.extractRowData("Fred,tanks,476,false", ","), + EmbedData.extractRowData("Stan,,-2", ",") + ) + + val actual = + DataType.matchHeaderRowLength(rows.map(_.toArray).toArray).map(_.toList).toList + + val expected = + List( + List( + DataType.StringData("name", false), + DataType.StringData("game", false), + DataType.StringData("highscore", false), + DataType.StringData("allowed", false) + ), + List( + DataType.StringData("bob", false), + DataType.NullData, + DataType.NullData, + DataType.NullData + ), + List( + DataType.StringData("Fred", false), + DataType.StringData("tanks", false), + DataType.IntData(476, false), + DataType.BooleanData(false, false) + ), + List( + DataType.StringData("Stan", false), + DataType.NullData, + DataType.IntData(-2, false), + DataType.NullData + ) + ) + + assertEquals(actual, expected) + + } + } diff --git a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala index 7ca94ec2d..8da4ee1fe 100644 --- a/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala +++ b/indigo-plugin/indigo-plugin/test/src/indigoplugin/generators/GeneratorAcceptanceTests.scala @@ -40,10 +40,10 @@ class GeneratorAcceptanceTests extends munit.FunSuite { | |// DO NOT EDIT: Generated by Indigo. | - |enum StatsEnum(val level: Int, val bonus: Int): - | case Intelligence extends StatsEnum(2, 4) - | case Strength extends StatsEnum(10, 0) - | case Fortitude extends StatsEnum(4, 1) + |enum StatsEnum(val level: Int, val bonus: Int, val stackable: Option[Boolean]): + | case Intelligence extends StatsEnum(2, 4, Some(true)) + | case Strength extends StatsEnum(10, 0, None) + | case Fortitude extends StatsEnum(4, 1, Some(false)) |""".stripMargin assertEquals(actual.trim, expected.trim) @@ -72,13 +72,13 @@ class GeneratorAcceptanceTests extends munit.FunSuite { | |// DO NOT EDIT: Generated by Indigo. | - |final case class StatsMap(val level: Int, val bonus: Int, val code: String) + |final case class StatsMap(level: Int, bonus: Int, code: Option[String]) |object StatsMap: | val data: Map[String, StatsMap] = | Map( - | "intelligence" -> StatsMap(2, 4, "i"), - | "strength" -> StatsMap(10, 0, "st"), - | "fortitude" -> StatsMap(4, 1, "_frt") + | "intelligence" -> StatsMap(2, 4, Some("i")), + | "strength" -> StatsMap(10, 0, None), + | "fortitude" -> StatsMap(4, 1, Some("_frt")) | ) |""".stripMargin @@ -144,9 +144,9 @@ class GeneratorAcceptanceTests extends munit.FunSuite { |// DO NOT EDIT: Generated by Indigo. |/* |"name": String,"level": String,"bonus": String,"code": String - |"intelligence": String,2: Int,4: Int,"i": String - |"strength": String,10: Int,0: Int,"st": String - |"fortitude": String,4: Int,1: Int,"_frt": String + |"intelligence": String,2: Int,4: Int,Some("i"): Option[String] + |"strength": String,10: Int,0: Int,None: Option[Any] + |"fortitude": String,4: Int,1: Int,Some("_frt"): Option[String] |*/ |""".stripMargin diff --git a/indigo-plugin/test-assets/data/stats.csv b/indigo-plugin/test-assets/data/stats.csv index e0b053e51..8f5d6dcba 100644 --- a/indigo-plugin/test-assets/data/stats.csv +++ b/indigo-plugin/test-assets/data/stats.csv @@ -1,5 +1,5 @@ -name,level,bonus -intelligence,2,4 -strength,10,0 -fortitude,4,1 +name,level,bonus,stackable +intelligence,2,4,true +strength,10,0, +fortitude,4,1,false diff --git a/indigo-plugin/test-assets/data/stats.md b/indigo-plugin/test-assets/data/stats.md index 8674df389..5dacb745f 100644 --- a/indigo-plugin/test-assets/data/stats.md +++ b/indigo-plugin/test-assets/data/stats.md @@ -1,5 +1,5 @@ | name | level | bonus | code | | ------------ | ----- | ----- | ---- | | intelligence | 2 | 4 | i | -| strength | 10 | 0 | st | +| strength | 10 | 0 | | | fortitude | 4 | 1 | _frt |