Merge pull request #73 from shazam/retry

Allow 'retry' of failing tests.

README.md

@@ -69,16 +69,19 @@ Will potentially be unsupported, as it's the least developer friendly. Check out
 ```
 > gradlew fork-runner:run -Pargs='ARGUMENTS LIST'
 
-With the below options.       The APK and test APK parameters are mandatory:
-    --sdk                     Path to Android SDK. Defaults to the ANDROID_HOME environment variable.
-    --apk                     Path to application. This parameter is required.
-    --test-apk                Path to test application. This parameter is required.
-    --output                  Output path. Defaults to "fork-output"
-    --test-package            The package to consider when finding tests to run. Defaults to instrumentation package.
-    --test-class-regex        Regex determining class names to consider when finding tests to run. Defaults to ^((?!Abstract).)*Test$
-    --test-timeout            The maximum amount of time during which the tests are allowed to not output any response, in milliseconds
-    --fail-on-failure         Non-zero exit code on failure. Defaults to false.
-    --fallback-to-screenshots If a device does not support videos, define if you'd like animated GIFs (experimental). Defaults to true.
+With the below options.         The APK and test APK parameters are mandatory:
+    --sdk                       Path to Android SDK. Defaults to the ANDROID_HOME environment variable.
+    --apk                       Path to application. This parameter is required.
+    --test-apk                  Path to test application. This parameter is required.
+    --output                    Output path. Defaults to "fork-output"
+    --test-package              The package to consider when finding tests to run. Defaults to instrumentation package.
+    --test-class-regex          Regex determining class names to consider when finding tests to run. Defaults to ^((?!Abstract).)*Test$
+    --test-timeout              The maximum amount of time during which the tests are allowed to not output any response, in milliseconds
+    --fail-on-failure           Non-zero exit code on failure. Defaults to false.
+    --fallback-to-screenshots   If a device does not support videos, define if you'd like animated GIFs (experimental). Defaults to true.
+    --total-allowed-retry-quota Total amount of allowed retries. If a test case fails and this quota hasn't been exhausted yet,
+                                the test case is scheduled to be executed again in the same device pool. Default to 0;
+    --retry-per-test-case-quota Amount of times a single test can be re-executed before declaring it a failure. Default to 1.
 ```
 
 For example:

fork-common/src/main/java/com/shazam/fork/summary/Summary.java

@@ -23,6 +23,7 @@ public class Summary {
     private final String title;
     private final String subtitle;
     private final ArrayList<String> ignoredTests;
+    private final ArrayList<String> failedTests;
 
     @Nonnull
     public List<PoolSummary> getPoolSummaries() {
@@ -42,11 +43,16 @@ public ArrayList<String> getIgnoredTests() {
         return ignoredTests;
     }
 
+    public ArrayList<String> getFailedTests() {
+        return failedTests;
+    }
+
     public static class Builder {
         private final List<PoolSummary> poolSummaries = new ArrayList<>();
         private final ArrayList<String> ignoredTests = new ArrayList<>();
         private String title = "Report Title";
         private String subtitle = "Report Subtitle";
+        private ArrayList<String> failedTests =  new ArrayList<>();
 
         public static Builder aSummary() {
             return new Builder();
@@ -72,6 +78,11 @@ public Builder addIgnoredTest(String s) {
             return this;
         }
 
+        public Builder addFailedTests(String failedTests) {
+            this.failedTests.add(failedTests);
+            return this;
+        }
+
         public Summary build() {
             return new Summary(this);
         }
@@ -82,5 +93,6 @@ private Summary(Builder builder) {
         title = builder.title;
         subtitle = builder.subtitle;
         ignoredTests = builder.ignoredTests;
+        failedTests = builder.failedTests;
     }
 }

fork-common/src/main/java/com/shazam/fork/summary/TestResult.java

@@ -14,17 +14,23 @@
 
 import com.shazam.fork.model.Device;
 
+import java.util.HashMap;
+import java.util.Map;
+
 import javax.annotation.Nonnull;
 
 import static com.google.common.base.Strings.isNullOrEmpty;
 
 public class TestResult {
+    public static final String SUMMARY_KEY_TOTAL_FAILURE_COUNT = "totalFailureCount";
+
     private final Device device;
     private final float timeTaken;
     private final String testClass;
     private final String testMethod;
     private final String errorTrace;
     private final String failureTrace;
+    private final Map<String, String> testMetrics;
 
     public Device getDevice() {
         return device;
@@ -42,6 +48,16 @@ public String getTestMethod() {
         return testMethod;
     }
 
+    public int getTotalFailureCount() {
+        int result = 0;
+
+        if (testMetrics != null
+                && testMetrics.containsKey(SUMMARY_KEY_TOTAL_FAILURE_COUNT)) {
+            result = Integer.parseInt(testMetrics.get(SUMMARY_KEY_TOTAL_FAILURE_COUNT));
+        }
+        return result;
+    }
+
     @Nonnull
     public ResultStatus getResultStatus() {
         if (!isNullOrEmpty(errorTrace)) {
@@ -71,6 +87,7 @@ public static class Builder {
         private String testMethod;
         private String errorTrace;
         private String failureTrace;
+        private Map<String, String> testMetrics = new HashMap<>();
 
         public static Builder aTestResult() {
             return new Builder();
@@ -110,6 +127,12 @@ public Builder withFailureTrace(String trace) {
             return this;
         }
 
+        public Builder withTestMetrics(Map<String, String> testMetrics) {
+            this.testMetrics.clear();
+            this.testMetrics.putAll(testMetrics);
+            return this;
+        }
+
         public TestResult build() {
             return new TestResult(this);
         }
@@ -123,5 +146,6 @@ private TestResult(Builder builder) {
         testMethod = builder.testMethod;
         errorTrace = builder.errorTrace;
         failureTrace = builder.failureTrace;
+        this.testMetrics = builder.testMetrics;
     }
 }

fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkExtension.groovy

@@ -46,4 +46,14 @@ class ForkExtension {
      * Indicate that screenshots are allowed when videos are not supported.
      */
     boolean fallbackToScreenshots
+
+    /**
+     * Amount of re-executions of failing tests allowed.
+     */
+    int totalAllowedRetryQuota
+
+    /**
+     * Max number of time each testCase is attempted again before declaring it as a failure.
+     */
+    int retryPerTestCaseQuota
 }

fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkPlugin.groovy

@@ -82,6 +82,8 @@ class ForkPlugin implements Plugin<Project> {
                 ignoreFailures = config.ignoreFailures
                 testOutputTimeout = config.testOutputTimeout
                 fallbackToScreenshots = config.fallbackToScreenshots;
+                totalAllowedRetryQuota = config.totalAllowedRetryQuota;
+                retryPerTestCaseQuota = config.retryPerTestCaseQuota;
 
                 dependsOn projectOutput.assemble, variant.assemble
             }

fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkRunTask.groovy

@@ -55,6 +55,10 @@ class ForkRunTask extends DefaultTask implements VerificationTask {
 
     boolean fallbackToScreenshots;
 
+    int totalAllowedRetryQuota;
+
+    int retryPerTestCaseQuota;
+
     @TaskAction
     void runFork() {
         LOG.info("Run instrumentation tests $instrumentationApk for app $applicationApk")
@@ -70,6 +74,8 @@ class ForkRunTask extends DefaultTask implements VerificationTask {
                 .withTestPackage(testPackage)
                 .withTestOutputTimeout(testOutputTimeout)
                 .withFallbackToScreenshots(fallbackToScreenshots)
+                .withTotalAllowedRetryQuota(totalAllowedRetryQuota)
+                .withRetryPerTestCaseQuota(retryPerTestCaseQuota)
 
         boolean success = fork.build().run()
 

fork-reporter/src/main/java/com/shazam/fork/reporter/FlakinessSorter.java

@@ -89,9 +89,8 @@ private List<UnsortedPoolHistory> reduceToUnsortedHistories(
                             .build();
                     testLabels.add(testLabel);
 
-                    ResultStatus resultStatus = testResult.getResultStatus();
                     TestInstance testInstance = testInstance()
-                            .withResultStatus(resultStatus)
+                            .withResultStatusFrom(testResult)
                             .withLink(testLinkCreator.createLinkToTest(buildLink, poolSummary.getPoolName(), testLabel))
                             .build();
                     table.put(testLabel, build, testInstance);
@@ -131,7 +130,7 @@ private TreeBasedTable<ScoredTestLabel, Build, TestInstance> sortTable(Table<Tes
 
         TreeBasedTable<ScoredTestLabel, Build, TestInstance> sortedTable = create(
                 (scoredTest1, scoredTest2) -> scoredTest1.getTestScore().compareTo(scoredTest2.getTestScore()),
-                (build1, build2) -> build1.getBuildId().compareTo(build2.getBuildId()));
+                Build::compareTo);
 
         for (TestLabel testLabel : testLabelsFullIndex) {
             List<TestInstance> testInstances = collectInstancesOfTest(rawResultsTable, buildsFullIndex, testLabel);

fork-reporter/src/main/java/com/shazam/fork/reporter/model/Build.java

@@ -10,7 +10,12 @@
 
 package com.shazam.fork.reporter.model;
 
-public class Build {
+import javax.annotation.Nonnull;
+
+import static java.lang.Integer.compare;
+import static java.lang.Integer.parseInt;
+
+public class Build implements Comparable<Build> {
     private final String buildId;
     private final String link;
 
@@ -27,6 +32,20 @@ private Build(Builder builder) {
         this.link = builder.link;
     }
 
+    @Override
+    public int compareTo(@Nonnull Build other) {
+        if (this == other) {
+            return 0;
+        }
+
+        if (tryParseInt(getBuildId())
+                && tryParseInt(other.getBuildId())) {
+            return compare(parseInt(getBuildId()), parseInt(other.getBuildId()));
+        } else {
+            return getBuildId().compareTo(other.getBuildId());
+        }
+    }
+
     public static class Builder {
         private String buildId;
         private String link;
@@ -49,4 +68,14 @@ public Build build() {
             return new Build(this);
         }
     }
+
+    private boolean tryParseInt(String value) {
+        try {
+            //noinspection ResultOfMethodCallIgnored
+            parseInt(value);
+            return true;
+        } catch (NumberFormatException | NullPointerException e) {
+            return false;
+        }
+    }
 }

fork-reporter/src/main/java/com/shazam/fork/reporter/model/Status.java

@@ -13,5 +13,6 @@
 public enum Status {
     PASS,
     FAIL,
-    MISSING
+    MISSING,
+    WARN
 }

fork-reporter/src/main/java/com/shazam/fork/reporter/model/TestInstance.java

@@ -11,6 +11,7 @@
 package com.shazam.fork.reporter.model;
 
 import com.shazam.fork.summary.ResultStatus;
+import com.shazam.fork.summary.TestResult;
 
 import javax.annotation.Nonnull;
 
@@ -42,8 +43,12 @@ public static Builder testInstance() {
             return new Builder();
         }
 
-        public Builder withResultStatus(ResultStatus resultStatus) {
-            this.resultStatus = fromResultStatus(resultStatus);
+        public Builder withResultStatusFrom(TestResult resultStatus) {
+            Status status = fromResultStatus(resultStatus.getResultStatus());
+            if( status == PASS && resultStatus.getTotalFailureCount() > 0){
+                status = WARN;
+            }
+            this.resultStatus = status;
             return this;
         }
 

fork-reporter/src/main/resources/static/pool-flakiness.css

@@ -78,6 +78,14 @@ body {
   background-color: #99c999;
 }
 
+.birds-eye .test.warn {
+  background-color: #ffffcc;
+}
+
+.birds-eye .test.warn:hover {
+  background-color: #f2f28b;
+}
+
 .birds-eye .test.fail {
   background-color: #e9b9b9;
 }

fork-runner/build.gradle

@@ -35,6 +35,9 @@ dependencies {
     testCompile("junit:junit:$JUNIT_VERSION") {
         exclude module:'hamcrest-core'
     }
+    testCompile(
+            "org.jmock:jmock:2.8.2",
+            "org.jmock:jmock-junit4:2.8.2")
 }
 
 jar {

fork-runner/src/main/java/com/shazam/fork/Configuration.java

@@ -30,6 +30,8 @@ public class Configuration {
     private final String testPackage;
     private final int testOutputTimeout;
     private final boolean fallbackToScreenshots;
+    private final int totalAllowedRetryQuota;
+    private final int retryPerTestCaseQuota;
 
     public Configuration(@Nonnull File androidSdk,
                          @Nonnull File applicationApk,
@@ -40,7 +42,9 @@ public Configuration(@Nonnull File androidSdk,
                          @Nonnull Pattern testPackagePattern,
                          @Nonnull String testPackage,
                          int testOutputTimeout,
-                         boolean fallbackToScreenshots) {
+                         boolean fallbackToScreenshots,
+                         int totalAllowedRetryQuota,
+                         int retryPerTestCaseQuota) {
         this.androidSdk = androidSdk;
         this.applicationApk = applicationApk;
         this.instrumentationApk = instrumentationApk;
@@ -51,6 +55,8 @@ public Configuration(@Nonnull File androidSdk,
         this.testPackage = testPackage;
         this.testOutputTimeout = testOutputTimeout;
         this.fallbackToScreenshots = fallbackToScreenshots;
+        this.totalAllowedRetryQuota = totalAllowedRetryQuota;
+        this.retryPerTestCaseQuota = retryPerTestCaseQuota;
     }
 
     @Nonnull
@@ -99,4 +105,12 @@ public int getTestOutputTimeout() {
     public boolean canFallbackToScreenshots() {
         return fallbackToScreenshots;
     }
+
+    public int getTotalAllowedRetryQuota() {
+        return totalAllowedRetryQuota;
+    }
+
+    public int getRetryPerTestCaseQuota() {
+        return retryPerTestCaseQuota;
+    }
 }