fix: resolve flaky tests counts discrepancy (Flank#2536)

invertase · Nov 21, 2024 · 923d791 · 923d791
1 parent d2f3a51
commit 923d791
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 0 deletions.
diff --git a/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestMerge.kt b/test_runner/src/main/kotlin/ftl/domain/junit/JUnitTestMerge.kt
@@ -47,6 +47,13 @@ fun JUnitTest.Suite.merge(other: JUnitTest.Suite): JUnitTest.Suite {
     this.errors = mergeInt(this.errors, other.errors)
     this.skipped = mergeInt(this.skipped, other.skipped)
     this.time = mergeDouble(this.time, other.time)
+    if (other.flakes != null) {
+        this.flakes = mergeInt(
+            this.flakes?.toString(),
+            other.flakes?.toString()
+        ).toInt()
+    }
+
 
     if (this.testcases == null) this.testcases = mutableListOf()
     if (other.testcases?.isNotEmpty() == true) {

diff --git a/test_runner/src/test/kotlin/ftl/client/xml/JUnitXmlTest.kt b/test_runner/src/test/kotlin/ftl/client/xml/JUnitXmlTest.kt
@@ -30,6 +30,15 @@ class JUnitXmlTest {
         <skipped/>
     </testcase>
     <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="0.001"/>
+</testsuite>
+        """.trimIndent()
+        val flakyTestSuiteXml = """
+<testsuite name="" tests="2" failures="0" flakes="1" errors="0" skipped="0" time="0.026" timestamp="2018-10-26T19:57:28" hostname="localhost">
+    <properties/>
+    <testcase name="testFails" classname="com.example.app.ExampleUiTest" time="0.0">
+        <skipped/>
+    </testcase>
+    <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="0.001"/>
 </testsuite>
         """.trimIndent()
     }
@@ -525,6 +534,32 @@ junit.framework.Assert.fail(Assert.java:50)</failure>
         val oneSuiteXml = parseOneSuiteXml(crashingOneSuiteMessage.writeToTempFile()).toXmlString().trimIndent()
         Assert.assertEquals("One Suite Messages should be the same!", expectedOneSuiteMessage, oneSuiteXml)
     }
+
+    @Test
+    fun `merge flakes`() {
+        val merged = parseOneSuiteXml(flakyTestSuiteXml.writeToTempFile())
+        merged.merge(merged)
+        val actual = merged.toXmlString().normalizeLineEnding()
+
+        assertThat(actual).isEqualTo(
+            """
+<?xml version='1.0' encoding='UTF-8' ?>
+<testsuites>
+  <testsuite name="" tests="4" failures="0" flakes="2" errors="0" skipped="0" time="0.052" timestamp="2018-10-26T19:57:28" hostname="localhost">
+    <testcase name="testFails" classname="com.example.app.ExampleUiTest" time="0.0">
+      <skipped/>
+    </testcase>
+    <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="0.001"/>
+    <testcase name="testFails" classname="com.example.app.ExampleUiTest" time="0.0">
+      <skipped/>
+    </testcase>
+    <testcase name="testPasses" classname="com.example.app.ExampleUiTest" time="0.001"/>
+  </testsuite>
+</testsuites>
+
+            """.trimIndent()
+        )
+    }
 }
 
 private fun String.writeToTempFile(): File = File.createTempFile("temp", "test")