Skip to content

Commit

Permalink
add summary and histogram
Browse files Browse the repository at this point in the history
  • Loading branch information
ailrst committed Jul 11, 2024
1 parent 996dfcb commit 9fde316
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 9 deletions.
8 changes: 4 additions & 4 deletions .github/workflows/run-examples.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,13 @@ jobs:

- name: System Tests
run: ./mill test.testOnly '*SystemTests*' || true

- uses: actions/upload-artifact@v4
with:
name: testresult-${{ github.run_number }}
path: src/test/*.csv

path: |
src/test/*.csv
src/test/*.svg
- run: |
tail -n+1 src/test/summary-*.csv
paste src/test/headers.md.part src/test/summary-*.md.part > $GITHUB_STEP_SUMMARY
paste src/test/headers.md.part src/test/summary-*.md.part | sed -s "s@HISTOtestresult-GTIRBHISTO@$(curl -F'file=@src/test/verifyTime-testresult-GTIRB.svg' -Fexpires=1920 http://0x0.st)@" | sed -s "s@HISTOtestresult-BAPHISTO@$(curl -F'file=@src/test/verifyTime-testresult-BAP.svg' -Fexpires=1920 http://0x0.st)@" > $GITHUB_STEP_SUMMARY
5 changes: 4 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,7 @@ examplesold/
src/test/scala/dump/
src/test/analysis/dump/
*.gtirb
*.json
*.json
src/test/*.svg
src/test/*.md.part
src/test/summary-*.md
117 changes: 113 additions & 4 deletions src/test/scala/SystemTests.scala
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import org.scalatest.funsuite.AnyFunSuite
import util.{Logger, PerformanceTimer}


import Numeric.Implicits._
import java.io.{BufferedWriter, File, FileWriter}
import scala.collection.mutable
import scala.io.Source
Expand All @@ -11,6 +11,8 @@ import scala.sys.process.*
* directory structure and file-name patterns.
*/



trait SystemTests extends AnyFunSuite {
val testPath = "./src/test/"
val correctPath = "./src/test/correct"
Expand Down Expand Up @@ -49,13 +51,19 @@ trait SystemTests extends AnyFunSuite {
val csv: String = "testCase," + TestResult.csvHeader + System.lineSeparator() + testResults.map(r => s"${r._1},${r._2.toCsv}").mkString(System.lineSeparator())
log(csv, testPath + "full-" + filename + ".csv")

val verifTimes = testResults.map(_._2.verifyTime.toDouble)

val numVerified = testResults.count(_._2.verified)
val numCounterexample = testResults.count(x => !x._2.verified && !x._2.timedOut)
val numSuccess = testResults.count(_._2.passed)
val numFail = testResults.count(!_._2.passed)
val numTimeout = testResults.count(_._2.timedOut)
val verifying = testResults.filter(x => !x._2.timedOut && x._2.verified).map(_._2.verifyTime)
val counterExamples = testResults.filter(x => !x._2.timedOut && !x._2.verified).map(_._2.verifyTime)
val medianVerifyTime = median(verifTimes)
val meanVerifyTime = mean(verifTimes)
val stdDevVerifyTime = stdDev(verifTimes)


info(s"Test summary: $numSuccess succeeded, $numFail failed: $numVerified verified, $numCounterexample did not verify (including $numTimeout timeouts).")
if (verifying.nonEmpty)
Expand All @@ -71,27 +79,41 @@ trait SystemTests extends AnyFunSuite {
"timeoutCount" -> numTimeout,
"verifyTotalTime" -> verifying.sum,
"counterexampleTotalTime" -> counterExamples.sum,
"meanVerifyTime" -> meanVerifyTime.toInt,
"medianVerifyTime" -> medianVerifyTime.toInt,
"stdDevVerifyTime" -> stdDevVerifyTime.toInt,
)
val summaryHeader = summaryMap.keys.mkString(",") + System.lineSeparator
val summaryRow = summaryMap.values.mkString(",") + System.lineSeparator
log(summaryHeader + summaryRow, testPath + "summary-" + filename + ".csv")

// generates a verification time histogram
val minB = Seq(0, meanVerifyTime - 2.25 * stdDevVerifyTime, verifTimes.min).max
val maxB = Seq(meanVerifyTime + 2.25 * stdDevVerifyTime, verifTimes.max).min
val nbins = 50

val histo = mkHistogram(nbins, Some(minB, maxB))(verifTimes.toSeq)
val svgHistogram = histoToSvg(filename, 400,300, histo)
log(svgHistogram, testPath + "verifyTime-" + filename + ".svg")


// generates a markdown table in separate parts.
// the complete markdown file can be constructed by horizontal (line-wise)
// concatenation of leftMarkdown and one or more partMarkdown.
val leftMarkdown =
val mdMap = summaryMap + ("VerifyTimeHistogram" -> ("![](" + "HISTO" + filename + "HISTO" + ")"))
val leftMarkdown =
s"""
|| Metric |
||--------|
|""".stripMargin
+ summaryMap.map((k,_) => s"| $k |${System.lineSeparator}").mkString
+ mdMap.map((k,_) => s"| $k |${System.lineSeparator}").mkString

val partMarkdown =
s"""
| $filename |
|-------|
|""".stripMargin
+ summaryMap.map((k,v) => s" $v |${System.lineSeparator}").mkString
+ mdMap.map((k,v) => s" $v |${System.lineSeparator}").mkString

val summaryMarkdown = leftMarkdown.linesIterator
.zip(partMarkdown.linesIterator)
Expand Down Expand Up @@ -202,6 +224,68 @@ trait SystemTests extends AnyFunSuite {

}



def histoToSvg(title: String, imgWidth: Int, imgHeight: Int, bins: List[((Double, Double), Int)]) : String = {
def template(width: Int = 300, height: Int = 130, content: String) =
s""" <svg width="${width}" height="${height}" xmlns="http://www.w3.org/2000/svg">
${content}
</svg> """
def mkRect(width: Int, height: Int, x: Int, y: Int, crx: Int=0, cry: Int=0, fill: String="Black") = {
s"""<rect width="$width" height="$height" x="$x" y="$y" rx="$crx" ry="$cry" fill="$fill" />"""
}
def text(content: String, x: Int, y: Int, cssClass: String = "small") = {
s"""<text x="$x" y="$y" class="$cssClass">$content</text>"""
}


val leftMargin = 20
val histWidth = imgWidth - leftMargin
val bottomMargin = 20
val topMargin = 20
val histHeight = imgHeight - topMargin - bottomMargin
val maxHeight = bins.map(_._2).max
val binWidth : Double = (histWidth).doubleValue / bins.size
val heightScaling : Double = (histHeight.doubleValue)/(maxHeight)
val binPos = (0 to bins.size).map(i => (leftMargin + i * binWidth, binWidth * (i + 1)))
.zip(bins.map((_, bh) => heightScaling * bh))

val rects = binPos.map((binXX, height) =>
mkRect(binWidth.ceil.intValue, height.intValue, binXX._1.floor.intValue, histHeight.intValue - height.intValue + topMargin))

val labels = {
(text(title, imgWidth / 8, topMargin - 5),
text("0", 0, histHeight + topMargin),
text(maxHeight.toInt.toString, 0, topMargin),
text(bins.head._1._1.toInt.toString, 0, imgHeight),
text(bins.last._1._2.toInt.toString, (binWidth*(bins.size)).intValue - leftMargin, imgHeight))
}

val content = (rects ++ labels.toList).mkString("\n")
template(imgWidth, imgHeight, content)
}


def loadHisto() = {
val source = scala.io.Source.fromFile("src/test/full-testresult-GTIRB.csv").getLines().toList
val headers = source.head.split(",")

val res = headers.map(h => h -> mutable.ArrayBuffer[String]()).toMap[String, mutable.ArrayBuffer[String]]

source.tail.map(line => {
val cols = line.split(",")
headers.zip(cols).foreach((h,v) => res(h).append(v))
})

val timeValues = res("verifyTime").map(_.toDouble)
val histo = mkHistogram(50, Some(800.0, 1000.0))(timeValues.toSeq)


println(histoToSvg("test histogram", 500, 300, histo))
}



class SystemTestsBAP extends SystemTests {
runTests(correctPrograms, correctPath, "correct", true, true)
runTests(incorrectPrograms, incorrectPath, "incorrect", false, true)
Expand All @@ -217,3 +301,28 @@ class SystemTestsGTIRB extends SystemTests {
summary("testresult-GTIRB")
}
}



// https://stackoverflow.com/questions/39617213/scala-what-is-the-generic-way-to-calculate-standard-deviation
def mean[T: Numeric](xs: Iterable[T]): Double = xs.sum.toDouble / xs.size

def variance[T: Numeric](xs: Iterable[T]): Double = {
val avg = mean(xs)

xs.map(_.toDouble).map(a => math.pow(a - avg, 2)).sum / xs.size
}

def median(xs: Iterable[Double]) = xs.toArray.sorted.apply(xs.size / 2)

def stdDev[T: Numeric](xs: Iterable[T]): Double = math.sqrt(variance(xs))

// https://stackoverflow.com/questions/24536215/scala-simple-histogram
def mkHistogram(n_bins: Int, lowerUpperBound: Option[(Double, Double)] = None)(xs: Seq[Double]) : List[((Double, Double), Int)] = {
val (mn, mx) = lowerUpperBound getOrElse(xs.min, xs.max)
val epsilon = 0.0001
val binSize = (mx - mn) / n_bins * (1 + epsilon)
val bins = (0 to n_bins).map(mn + _ * binSize).sliding(2).map(xs => (xs(0), xs(1)))
def binContains(bin:(Double,Double),x: Double) = (x >= bin._1) && (x < bin._2)
bins.map(bin => (bin, xs.count(binContains(bin,_)))).toList
}

0 comments on commit 9fde316

Please sign in to comment.