Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Generators to support Optional types #637

Merged
merged 5 commits into from
Nov 19, 2023
Merged

Generators to support Optional types #637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
94e9497
Null support, ignore delimiters start/end of line
davesmith00000 Nov 18, 2023
656f18d
Empty cells are 'null', "null" is a string
davesmith00000 Nov 19, 2023
0fca5df
Handle negative ints and doubles
davesmith00000 Nov 19, 2023
459a775
Acceptance tests failing
davesmith00000 Nov 19, 2023
5273640
Tests pass with optional fields
davesmith00000 Nov 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
229 changes: 177 additions & 52 deletions indigo-plugin/indigo-plugin/src/indigoplugin/generators/EmbedData.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ object EmbedData {
os.read.lines(filePath).filter(rowFilter)
}

val rows =
lines.map(row => extractRowData(row, delimiter)).toList

val dataFrame =
DataFrame.fromRows(
lines.map(row => extractRowData(row, delimiter)).toList
)
DataFrame.fromRows(rows)

val wd = outDir / Generators.OutputDirName

Expand Down Expand Up @@ -74,16 +75,35 @@ object EmbedData {
Seq(file)
}

def extractRows(rows: List[String], delimiter: String): List[List[DataType]] =
rows.map(r => extractRowData(r, delimiter))
def extractRowData(row: String, delimiter: String): List[DataType] = {

val cleanDelimiter: String =
if (delimiter == "\\|") "|" else delimiter

val cleanRow: String =
row.trim match {
case r if r.startsWith(cleanDelimiter) && r.endsWith(cleanDelimiter) =>
r.drop(cleanDelimiter.length()).dropRight(cleanDelimiter.length())

case r if r.startsWith(cleanDelimiter) =>
r.drop(cleanDelimiter.length())

case r if r.endsWith(cleanDelimiter) =>
r.dropRight(cleanDelimiter.length())

def extractRowData(row: String, delimiter: String): List[DataType] =
parse(delimiter)(row).map(_._1).collect {
case d @ DataType.StringData(s) if s.nonEmpty => d
case d: DataType.BooleanData => d
case d: DataType.DoubleData => d
case d: DataType.IntData => d
case r =>
r
}

parse(delimiter)(cleanRow).map(_._1).collect {
case d @ DataType.StringData(s, _) if s.nonEmpty => d
case DataType.StringData(_, _) => DataType.NullData
case d: DataType.BooleanData => d
case d: DataType.DoubleData => d
case d: DataType.IntData => d
case DataType.NullData => DataType.NullData
}
}

// A parser of things,
// is a function from strings,
Expand Down Expand Up @@ -113,6 +133,9 @@ object EmbedData {

sealed trait DataType {

def nullable: Boolean
def makeOptional: DataType

def isString: Boolean =
this match {
case _: DataType.StringData => true
Expand All @@ -137,58 +160,116 @@ sealed trait DataType {
case _ => false
}

def isNull: Boolean =
this match {
case DataType.NullData => true
case _ => false
}

def toStringData: DataType.StringData =
this match {
case s: DataType.StringData => s
case DataType.BooleanData(value) => DataType.StringData(value.toString)
case DataType.DoubleData(value) => DataType.StringData(value.toString)
case DataType.IntData(value) => DataType.StringData(value.toString)
case s: DataType.StringData if s.nullable => DataType.StringData(s"""Some("${s.value}")""", true)
case s: DataType.StringData => s
case DataType.BooleanData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true)
case DataType.BooleanData(value, _) => DataType.StringData(value.toString, false)
case DataType.DoubleData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true)
case DataType.DoubleData(value, _) => DataType.StringData(value.toString, false)
case DataType.IntData(value, nullable) if nullable => DataType.StringData(s"Some(${value.toString})", true)
case DataType.IntData(value, _) => DataType.StringData(value.toString, false)
case DataType.NullData => DataType.StringData("None", true)
}

def asString: String =
this match {
case s: DataType.StringData => s""""${s.value}""""
case DataType.BooleanData(value) => value.toString
case DataType.DoubleData(value) => value.toString
case DataType.IntData(value) => value.toString
case s: DataType.StringData if s.nullable => s"""Some("${s.value}")"""
case s: DataType.StringData => s""""${s.value}""""
case DataType.BooleanData(value, nullable) if nullable => s"Some(${value.toString})"
case DataType.BooleanData(value, _) => value.toString
case DataType.DoubleData(value, nullable) if nullable => s"Some(${value.toString})"
case DataType.DoubleData(value, _) => value.toString
case DataType.IntData(value, nullable) if nullable => s"Some(${value.toString})"
case DataType.IntData(value, _) => value.toString
case DataType.NullData => "None"
}

def giveTypeName: String =
this match {
case _: DataType.StringData => "String"
case _: DataType.BooleanData => "Boolean"
case _: DataType.DoubleData => "Double"
case _: DataType.IntData => "Int"
case d: DataType.StringData if d.nullable => "Option[String]"
case _: DataType.StringData => "String"
case d: DataType.BooleanData if d.nullable => "Option[Boolean]"
case _: DataType.BooleanData => "Boolean"
case d: DataType.DoubleData if d.nullable => "Option[Double]"
case _: DataType.DoubleData => "Double"
case d: DataType.IntData if d.nullable => "Option[Int]"
case _: DataType.IntData => "Int"
case DataType.NullData => "Option[Any]"
}

}
object DataType {

// Most to least specific: Boolean, Int, Double, String
final case class BooleanData(value: Boolean) extends DataType
final case class IntData(value: Int) extends DataType {
def toDoubleData: DoubleData = DoubleData(value.toDouble)
final case class BooleanData(value: Boolean, nullable: Boolean) extends DataType {
def makeOptional: BooleanData = this.copy(nullable = true)
}
object BooleanData {
def apply(value: Boolean): BooleanData = BooleanData(value, false)
}

final case class IntData(value: Int, nullable: Boolean) extends DataType {
def toDoubleData: DoubleData = DoubleData(value.toDouble, nullable)
def makeOptional: IntData = this.copy(nullable = true)
}
object IntData {
def apply(value: Int): IntData = IntData(value, false)
}

final case class DoubleData(value: Double, nullable: Boolean) extends DataType {
def makeOptional: DoubleData = this.copy(nullable = true)
}
object DoubleData {
def apply(value: Double): DoubleData = DoubleData(value, false)
}

final case class StringData(value: String, nullable: Boolean) extends DataType {
def makeOptional: StringData = this.copy(nullable = true)
}
object StringData {
def apply(value: String): StringData = StringData(value, false)
}

case object NullData extends DataType {
val nullable: Boolean = true
def makeOptional: DataType = this
}
final case class DoubleData(value: Double) extends DataType
final case class StringData(value: String) extends DataType

private val isBoolean: Regex = """^(true|false)$""".r
private val isDouble: Regex = """^([0-9]+?)\.([0-9]+)$""".r
private val isInt: Regex = """^([0-9]+)$""".r
private val isInt: Regex = """^(\-?[0-9]+)$""".r
private val isDouble: Regex = """^(\-?[0-9]+?)\.([0-9]+)$""".r
private val isNull: Regex = """^$""".r

def decideType: String => DataType = {
case isBoolean(v) => BooleanData(v.toBoolean)
case isInt(v) => IntData(v.toInt)
case isDouble(v1, v2) => DoubleData(s"$v1.$v2".toDouble)
case v => StringData(v)
case isBoolean(v) => BooleanData(v.toBoolean, false)
case isInt(v) => IntData(v.toInt, false)
case isDouble(v1, v2) => DoubleData(s"$v1.$v2".toDouble, false)
case isNull(_) => NullData
case v => StringData(v, false)
}

def sameType(a: DataType, b: DataType): Boolean =
(a, b) match {
case (_: DataType.StringData, _: DataType.StringData) => true
case (DataType.NullData, _: DataType.StringData) => true
case (_: DataType.StringData, DataType.NullData) => true
case (_: DataType.BooleanData, _: DataType.BooleanData) => true
case (DataType.NullData, _: DataType.BooleanData) => true
case (_: DataType.BooleanData, DataType.NullData) => true
case (_: DataType.DoubleData, _: DataType.DoubleData) => true
case (DataType.NullData, _: DataType.DoubleData) => true
case (_: DataType.DoubleData, DataType.NullData) => true
case (_: DataType.IntData, _: DataType.IntData) => true
case (DataType.NullData, _: DataType.IntData) => true
case (_: DataType.IntData, DataType.NullData) => true
case _ => false
}

Expand All @@ -199,25 +280,52 @@ object DataType {
}

def allNumericTypes(l: List[DataType]): Boolean =
l.forall(d => d.isDouble || d.isInt)
l.forall(d => d.isDouble || d.isInt || d.isNull)

def hasOptionalValues(l: List[DataType]): Boolean =
l.contains(DataType.NullData)

def convertToBestType(l: List[DataType]): List[DataType] =
// Cases we can manage:
// - They're all the same!
// - They're all the same! Maybe optional...
// - Doubles and Ints, convert Ints to Doubles
// - Fallback is that everything is a string.
if (allSameType(l)) {
// All the same! Great!
l
} else if (allNumericTypes(l)) {
l.map {
case v @ DataType.DoubleData(_) => v
case v @ DataType.IntData(_) => v.toDoubleData
case v @ DataType.DoubleData(_, _) => v
case v @ DataType.IntData(_, _) => v.toDoubleData
case DataType.NullData => DataType.NullData
case s => throw new Exception(s"Unexpected non-numeric type '$s'") // Shouldn't get here.
}
} else {
// Nothing else to do, but make everything a string
l.map(_.toStringData)
// Nothing else to do, but make everything a string that isn't null.
l.map {
case DataType.NullData => DataType.NullData
case d => d.toStringData
}
}

def matchHeaderRowLength(rows: Array[Array[DataType]]): Array[Array[DataType]] =
rows.toList match {
case Nil =>
rows

case headers :: data =>
val l = headers.length
val res =
headers :: data.map { r =>
val diff = l - r.length
if (diff > 0) {
r ++ List.fill(diff)(DataType.NullData)
} else {
r
}
}

res.toArray
}

}
Expand All @@ -230,12 +338,26 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) {
data.tail

def alignColumnTypes: DataFrame = {
val transposed = rows.transpose
val stringKeys: Array[DataType] = transposed.head.map(_.toStringData)
val typeRows: Array[Array[DataType]] = transposed.tail
val columns =
DataType.matchHeaderRowLength(rows).transpose

val stringKeys: Array[DataType] =
columns.head.map(_.toStringData)

val typedColumns: Array[Array[DataType]] = columns.tail
.map(d => DataType.convertToBestType(d.toList).toArray)

val optionalColumns: Array[Array[DataType]] =
typedColumns.map { col =>
if (DataType.hasOptionalValues(col.toList)) {
col.map(_.makeOptional)
} else {
col
}
}

val cleanedRows: Array[Array[DataType]] =
(stringKeys +: typeRows).transpose
(stringKeys +: optionalColumns).transpose

this.copy(
data = headers.asInstanceOf[Array[DataType]] +: cleanedRows
Expand All @@ -257,10 +379,15 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) {
}
}

def renderVars: String = {
def renderVars(omitVal: Boolean): String = {
val names = headers.drop(1).map(_.value)
val types = rows.head.drop(1).map(_.giveTypeName)
names.zip(types).map { case (n, t) => s"val ${toSafeNameCamel(n)}: $t" }.mkString(", ")
names
.zip(types)
.map { case (n, t) =>
(if (omitVal) "" else "val ") + s"${toSafeNameCamel(n)}: $t"
}
.mkString(", ")
}

def renderEnum(moduleName: String, extendsFrom: Option[String]): String = {
Expand All @@ -280,7 +407,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) {
.getOrElse("")

s"""
|enum $moduleName(${renderVars})$extFrom:
|enum $moduleName(${renderVars(false)})$extFrom:
|${renderedRows}
|""".stripMargin
}
Expand All @@ -294,7 +421,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) {
.mkString(",\n")

s"""
|final case class $moduleName(${renderVars})
|final case class $moduleName(${renderVars(true)})
|object $moduleName:
| val data: Map[String, $moduleName] =
| Map(
Expand All @@ -309,7 +436,7 @@ final case class DataFrame(data: Array[Array[DataType]], columnCount: Int) {
object DataFrame {

private val standardMessage: String =
"Embedded data must have two rows (minimum) of the same length (two columns minimum). The first row is the headers / field names. The first column are the keys. Cells cannot be empty."
"Embedded data must have two rows (minimum) of the same length (two columns minimum). The first row is the headers / field names. The first column are the keys."

def fromRows(rows: List[List[DataType]]): DataFrame =
rows match {
Expand All @@ -319,15 +446,13 @@ object DataFrame {
case _ :: Nil =>
throw new Exception("Only one row of data found. " + standardMessage)

case h :: t =>
case h :: _ =>
val len = h.length

if (len == 0) {
throw new Exception("No data to create. " + standardMessage)
} else if (len == 1) {
throw new Exception("Only one column of data. " + standardMessage)
} else if (!t.forall(_.length == len)) {
throw new Exception(s"All rows must be the same length. Header row had '$len' columns. " + standardMessage)
} else {
DataFrame(rows.map(_.toArray).toArray, len).alignColumnTypes
}
Expand Down
Loading
Loading