fix(ios): add xcode timeout result parsing (#985)

* [fix] consider xcode timeout when parsing test run progress.

* Consider 1 minute timeout case

Author: msd117c

vendor/vendor-apple/base/src/main/kotlin/com/malinskiy/marathon/apple/logparser/parser/DeviceFailureParser.kt

@@ -4,7 +4,7 @@ import com.malinskiy.marathon.apple.logparser.TestEventProducer
 import com.malinskiy.marathon.apple.test.TestEvent
 import com.malinskiy.marathon.apple.test.TestRunFailed
 
-class DeviceFailureParser : com.malinskiy.marathon.apple.logparser.TestEventProducer {
+class DeviceFailureParser : TestEventProducer {
     private val patterns = listOf(
         "Failed to install or launch the test runner",
         "Software caused connection abort",

vendor/vendor-apple/base/src/main/kotlin/com/malinskiy/marathon/apple/logparser/parser/TestRunProgressParser.kt

@@ -14,7 +14,7 @@ import kotlin.math.roundToInt
 class TestRunProgressParser(
     private val timer: Timer,
     private val targetName: String,
-) : com.malinskiy.marathon.apple.logparser.TestEventProducer {
+) : TestEventProducer {
 
     val logger = MarathonLogging.logger(TestRunProgressParser::class.java.simpleName)
 
@@ -29,6 +29,11 @@ class TestRunProgressParser(
      * $4 = reason
      */
     val FAILING_TEST_MATCHER = "(/.+:\\d+):\\serror:\\s[\\+\\-]\\[(.*)\\s(.*)\\]\\s:(\\s.*)".toRegex()
+
+    /**
+     * Timeout case from https://developer.apple.com/documentation/xctest/xctestcase/3526064-executiontimeallowance
+     */
+    val TIMEOUT_TEST_MATCHER = """Test Case '-\[(.+) (.+)]' exceeded execution time allowance of (\d+) minutes*\. The test may have hung.*""".toRegex()
 
     private var failingTestLine: String? = null
 
@@ -37,6 +42,8 @@ class TestRunProgressParser(
             parseTestStarted(line)?.let { listOf(it) }
         } else if (line.matches(TEST_CASE_FINISHED)) {
             parseTestFinished(line)?.let { listOf(it) }
+        } else if (line.matches(TIMEOUT_TEST_MATCHER)) {
+            parseTestTimeout(line)?.let { listOf(it) }
         } else if (line.matches(FAILING_TEST_MATCHER)) {
             failingTestLine = line
             null
@@ -108,6 +115,40 @@ class TestRunProgressParser(
         return null
     }
 
+    private fun parseTestTimeout(line: String): TestEvent? {
+        val matchResult = TIMEOUT_TEST_MATCHER.find(line)
+        val pkgWithClass = matchResult?.groups?.get(1)?.value
+        var pkg: String? = null
+        var clazz: String? = null
+        if (pkgWithClass != null) {
+            if (pkgWithClass.contains('.')) {
+                pkg = pkgWithClass.substringBeforeLast('.', missingDelimiterValue = "")
+                clazz = pkgWithClass.substringAfter('.', missingDelimiterValue = pkgWithClass)
+            } else {
+                pkg = targetName
+                clazz = pkgWithClass
+            }
+        }
+
+        val method = matchResult?.groups?.get(2)?.value
+        val duration = matchResult?.groups?.get(3)?.value?.toFloat()?.times(60)
+
+        logger.debug { "Test $pkg.$clazz.$method finished with result <timeout> after $duration seconds" }
+
+        if (pkg != null && clazz != null && method != null && duration != null) {
+            val test = Test(pkg, clazz, method, emptyList())
+
+            val endTime = timer.currentTimeMillis()
+            val startTime = endTime - (duration * 1000).roundToInt()
+
+            val trace = failingTestLine?.let {
+                parseFailingTest(it)
+            }
+            return TestFailed(test, startTime, endTime, trace)
+        }
+        return null
+    }
+
     private fun parseTestStarted(line: String): TestStarted? {
         failingTestLine = null
         val matchResult = TEST_CASE_STARTED.find(line)

vendor/vendor-apple/base/src/test/kotlin/com/malinskiy/marathon/apple/logparser/parser/TestRunProgressParserTest.kt

@@ -4,6 +4,7 @@ import assertk.assertThat
 import assertk.assertions.isEqualTo
 import com.malinskiy.marathon.apple.test.TestEvent
 import com.malinskiy.marathon.time.Timer
+import org.amshove.kluent.internal.assertEquals
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.mockito.Mockito.reset
@@ -104,4 +105,21 @@ class TestRunProgressParserTest {
         assertThat(events.map { it.toString() }.reduce { acc, s -> acc + "\n" + s })
             .isEqualTo(javaClass.getResourceAsStream("/fixtures/test_output/patrol_1.expected").reader().readText().trimEnd())
     }
+
+    @Test
+    fun testSample5() {
+        val parser = TestRunProgressParser(mockTimer, "")
+
+        val events = mutableListOf<TestEvent>()
+        javaClass.getResourceAsStream("/fixtures/test_output/timeout_0.log.input").bufferedReader().use {
+            it.lines().forEach { line ->
+                parser.process(line)?.let {
+                    events.addAll(it)
+                }
+            }
+        }
+
+        assertThat(events.map { it.toString() }.reduce { acc, s -> acc + "\n" + s })
+            .isEqualTo(javaClass.getResourceAsStream("/fixtures/test_output/timeout_0.expected").reader().readText().trimEnd())
+    }
 }

vendor/vendor-apple/base/src/test/resources/fixtures/test_output/timeout_0.expected

@@ -0,0 +1,2 @@
+TestStarted(id=Test(pkg=sample_appUITests, clazz=MoreTests, method=testPresentModal, metaProperties=[]))
+TestFailed(id=Test(pkg=sample_appUITests, clazz=MoreTests, method=testPresentModal, metaProperties=[]), startTime=1537187516000, endTime=1537187696000, trace=null)

vendor/vendor-apple/base/src/test/resources/fixtures/test_output/timeout_0.log.input

@@ -0,0 +1,15 @@
+Test Case '-[sample_appUITests.MoreTests testPresentModal]' started.
+    t =     0.00s Start Test at 2018-09-15 01:08:51.148
+    t =     0.03s Set Up
+    t =     0.03s     Open com.agoda.sample-app
+    t =     0.06s         Launch com.agoda.sample-app
+    t =     2.61s             Wait for com.agoda.sample-app to idle
+    t =     3.85s Waiting 30.0s for Button to exist
+    t =     4.85s     Checking `Expect predicate `exists == 1` for object Button`
+    t =     4.86s Tap Button
+    t =     4.86s     Wait for com.agoda.sample-app to idle
+    t =     4.87s     Find the Button
+    t =     4.98s         Wait for com.agoda.sample-app to idle
+    t =     4.99s     Synthesize event
+    t =     5.07s     Wait for com.agoda.sample-app to idle
+Test Case '-[sample_appUITests.MoreTests testPresentModal]' exceeded execution time allowance of 3 minutes. The test may have hung; check Xcode's test report for additional diagnostics. If the test requires additional time, use -[XCTestCase executionTimeAllowance] or one of xcodebuild's command-line overrides.