diff --git a/README.md b/README.md index 0080df49..c2ba247f 100644 --- a/README.md +++ b/README.md @@ -69,16 +69,19 @@ Will potentially be unsupported, as it's the least developer friendly. Check out ``` > gradlew fork-runner:run -Pargs='ARGUMENTS LIST' -With the below options. The APK and test APK parameters are mandatory: - --sdk Path to Android SDK. Defaults to the ANDROID_HOME environment variable. - --apk Path to application. This parameter is required. - --test-apk Path to test application. This parameter is required. - --output Output path. Defaults to "fork-output" - --test-package The package to consider when finding tests to run. Defaults to instrumentation package. - --test-class-regex Regex determining class names to consider when finding tests to run. Defaults to ^((?!Abstract).)*Test$ - --test-timeout The maximum amount of time during which the tests are allowed to not output any response, in milliseconds - --fail-on-failure Non-zero exit code on failure. Defaults to false. - --fallback-to-screenshots If a device does not support videos, define if you'd like animated GIFs (experimental). Defaults to true. +With the below options. The APK and test APK parameters are mandatory: + --sdk Path to Android SDK. Defaults to the ANDROID_HOME environment variable. + --apk Path to application. This parameter is required. + --test-apk Path to test application. This parameter is required. + --output Output path. Defaults to "fork-output" + --test-package The package to consider when finding tests to run. Defaults to instrumentation package. + --test-class-regex Regex determining class names to consider when finding tests to run. Defaults to ^((?!Abstract).)*Test$ + --test-timeout The maximum amount of time during which the tests are allowed to not output any response, in milliseconds + --fail-on-failure Non-zero exit code on failure. Defaults to false. + --fallback-to-screenshots If a device does not support videos, define if you'd like animated GIFs (experimental). Defaults to true. + --total-allowed-retry-quota Total amount of allowed retries. If a test case fails and this quota hasn't been exhausted yet, + the test case is scheduled to be executed again in the same device pool. Default to 0; + --retry-per-test-case-quota Amount of times a single test can be re-executed before declaring it a failure. Default to 1. ``` For example: diff --git a/fork-common/src/main/java/com/shazam/fork/summary/Summary.java b/fork-common/src/main/java/com/shazam/fork/summary/Summary.java index 423a0011..a4338456 100755 --- a/fork-common/src/main/java/com/shazam/fork/summary/Summary.java +++ b/fork-common/src/main/java/com/shazam/fork/summary/Summary.java @@ -23,6 +23,7 @@ public class Summary { private final String title; private final String subtitle; private final ArrayList ignoredTests; + private final ArrayList failedTests; @Nonnull public List getPoolSummaries() { @@ -42,11 +43,16 @@ public ArrayList getIgnoredTests() { return ignoredTests; } + public ArrayList getFailedTests() { + return failedTests; + } + public static class Builder { private final List poolSummaries = new ArrayList<>(); private final ArrayList ignoredTests = new ArrayList<>(); private String title = "Report Title"; private String subtitle = "Report Subtitle"; + private ArrayList failedTests = new ArrayList<>(); public static Builder aSummary() { return new Builder(); @@ -72,6 +78,11 @@ public Builder addIgnoredTest(String s) { return this; } + public Builder addFailedTests(String failedTests) { + this.failedTests.add(failedTests); + return this; + } + public Summary build() { return new Summary(this); } @@ -82,5 +93,6 @@ private Summary(Builder builder) { title = builder.title; subtitle = builder.subtitle; ignoredTests = builder.ignoredTests; + failedTests = builder.failedTests; } } diff --git a/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java b/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java index ae697173..3ab84b86 100755 --- a/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java +++ b/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java @@ -14,17 +14,23 @@ import com.shazam.fork.model.Device; +import java.util.HashMap; +import java.util.Map; + import javax.annotation.Nonnull; import static com.google.common.base.Strings.isNullOrEmpty; public class TestResult { + public static final String SUMMARY_KEY_TOTAL_FAILURE_COUNT = "totalFailureCount"; + private final Device device; private final float timeTaken; private final String testClass; private final String testMethod; private final String errorTrace; private final String failureTrace; + private final Map testMetrics; public Device getDevice() { return device; @@ -42,6 +48,16 @@ public String getTestMethod() { return testMethod; } + public int getTotalFailureCount() { + int result = 0; + + if (testMetrics != null + && testMetrics.containsKey(SUMMARY_KEY_TOTAL_FAILURE_COUNT)) { + result = Integer.parseInt(testMetrics.get(SUMMARY_KEY_TOTAL_FAILURE_COUNT)); + } + return result; + } + @Nonnull public ResultStatus getResultStatus() { if (!isNullOrEmpty(errorTrace)) { @@ -71,6 +87,7 @@ public static class Builder { private String testMethod; private String errorTrace; private String failureTrace; + private Map testMetrics = new HashMap<>(); public static Builder aTestResult() { return new Builder(); @@ -110,6 +127,12 @@ public Builder withFailureTrace(String trace) { return this; } + public Builder withTestMetrics(Map testMetrics) { + this.testMetrics.clear(); + this.testMetrics.putAll(testMetrics); + return this; + } + public TestResult build() { return new TestResult(this); } @@ -123,5 +146,6 @@ private TestResult(Builder builder) { testMethod = builder.testMethod; errorTrace = builder.errorTrace; failureTrace = builder.failureTrace; + this.testMetrics = builder.testMetrics; } } diff --git a/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkExtension.groovy b/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkExtension.groovy index bcb40701..dde4e31b 100644 --- a/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkExtension.groovy +++ b/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkExtension.groovy @@ -46,4 +46,14 @@ class ForkExtension { * Indicate that screenshots are allowed when videos are not supported. */ boolean fallbackToScreenshots + + /** + * Amount of re-executions of failing tests allowed. + */ + int totalAllowedRetryQuota + + /** + * Max number of time each testCase is attempted again before declaring it as a failure. + */ + int retryPerTestCaseQuota } diff --git a/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkPlugin.groovy b/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkPlugin.groovy index 03e57568..19698bc9 100644 --- a/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkPlugin.groovy +++ b/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkPlugin.groovy @@ -82,6 +82,8 @@ class ForkPlugin implements Plugin { ignoreFailures = config.ignoreFailures testOutputTimeout = config.testOutputTimeout fallbackToScreenshots = config.fallbackToScreenshots; + totalAllowedRetryQuota = config.totalAllowedRetryQuota; + retryPerTestCaseQuota = config.retryPerTestCaseQuota; dependsOn projectOutput.assemble, variant.assemble } diff --git a/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkRunTask.groovy b/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkRunTask.groovy index 035bf465..ca078465 100644 --- a/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkRunTask.groovy +++ b/fork-gradle-plugin/src/main/groovy/com/shazam/fork/gradle/ForkRunTask.groovy @@ -55,6 +55,10 @@ class ForkRunTask extends DefaultTask implements VerificationTask { boolean fallbackToScreenshots; + int totalAllowedRetryQuota; + + int retryPerTestCaseQuota; + @TaskAction void runFork() { LOG.info("Run instrumentation tests $instrumentationApk for app $applicationApk") @@ -70,6 +74,8 @@ class ForkRunTask extends DefaultTask implements VerificationTask { .withTestPackage(testPackage) .withTestOutputTimeout(testOutputTimeout) .withFallbackToScreenshots(fallbackToScreenshots) + .withTotalAllowedRetryQuota(totalAllowedRetryQuota) + .withRetryPerTestCaseQuota(retryPerTestCaseQuota) boolean success = fork.build().run() diff --git a/fork-reporter/src/main/java/com/shazam/fork/reporter/FlakinessSorter.java b/fork-reporter/src/main/java/com/shazam/fork/reporter/FlakinessSorter.java index bcdadf7b..bfee5785 100644 --- a/fork-reporter/src/main/java/com/shazam/fork/reporter/FlakinessSorter.java +++ b/fork-reporter/src/main/java/com/shazam/fork/reporter/FlakinessSorter.java @@ -89,9 +89,8 @@ private List reduceToUnsortedHistories( .build(); testLabels.add(testLabel); - ResultStatus resultStatus = testResult.getResultStatus(); TestInstance testInstance = testInstance() - .withResultStatus(resultStatus) + .withResultStatusFrom(testResult) .withLink(testLinkCreator.createLinkToTest(buildLink, poolSummary.getPoolName(), testLabel)) .build(); table.put(testLabel, build, testInstance); @@ -131,7 +130,7 @@ private TreeBasedTable sortTable(Table sortedTable = create( (scoredTest1, scoredTest2) -> scoredTest1.getTestScore().compareTo(scoredTest2.getTestScore()), - (build1, build2) -> build1.getBuildId().compareTo(build2.getBuildId())); + Build::compareTo); for (TestLabel testLabel : testLabelsFullIndex) { List testInstances = collectInstancesOfTest(rawResultsTable, buildsFullIndex, testLabel); diff --git a/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Build.java b/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Build.java index e2e4da1e..66cb85a5 100644 --- a/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Build.java +++ b/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Build.java @@ -10,7 +10,12 @@ package com.shazam.fork.reporter.model; -public class Build { +import javax.annotation.Nonnull; + +import static java.lang.Integer.compare; +import static java.lang.Integer.parseInt; + +public class Build implements Comparable { private final String buildId; private final String link; @@ -27,6 +32,20 @@ private Build(Builder builder) { this.link = builder.link; } + @Override + public int compareTo(@Nonnull Build other) { + if (this == other) { + return 0; + } + + if (tryParseInt(getBuildId()) + && tryParseInt(other.getBuildId())) { + return compare(parseInt(getBuildId()), parseInt(other.getBuildId())); + } else { + return getBuildId().compareTo(other.getBuildId()); + } + } + public static class Builder { private String buildId; private String link; @@ -49,4 +68,14 @@ public Build build() { return new Build(this); } } + + private boolean tryParseInt(String value) { + try { + //noinspection ResultOfMethodCallIgnored + parseInt(value); + return true; + } catch (NumberFormatException | NullPointerException e) { + return false; + } + } } diff --git a/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Status.java b/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Status.java index f07bc644..a022361e 100644 --- a/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Status.java +++ b/fork-reporter/src/main/java/com/shazam/fork/reporter/model/Status.java @@ -13,5 +13,6 @@ public enum Status { PASS, FAIL, - MISSING + MISSING, + WARN } diff --git a/fork-reporter/src/main/java/com/shazam/fork/reporter/model/TestInstance.java b/fork-reporter/src/main/java/com/shazam/fork/reporter/model/TestInstance.java index 459ccff2..95302d4d 100644 --- a/fork-reporter/src/main/java/com/shazam/fork/reporter/model/TestInstance.java +++ b/fork-reporter/src/main/java/com/shazam/fork/reporter/model/TestInstance.java @@ -11,6 +11,7 @@ package com.shazam.fork.reporter.model; import com.shazam.fork.summary.ResultStatus; +import com.shazam.fork.summary.TestResult; import javax.annotation.Nonnull; @@ -42,8 +43,12 @@ public static Builder testInstance() { return new Builder(); } - public Builder withResultStatus(ResultStatus resultStatus) { - this.resultStatus = fromResultStatus(resultStatus); + public Builder withResultStatusFrom(TestResult resultStatus) { + Status status = fromResultStatus(resultStatus.getResultStatus()); + if( status == PASS && resultStatus.getTotalFailureCount() > 0){ + status = WARN; + } + this.resultStatus = status; return this; } diff --git a/fork-reporter/src/main/resources/static/pool-flakiness.css b/fork-reporter/src/main/resources/static/pool-flakiness.css index 8705e1c5..410f5861 100644 --- a/fork-reporter/src/main/resources/static/pool-flakiness.css +++ b/fork-reporter/src/main/resources/static/pool-flakiness.css @@ -78,6 +78,14 @@ body { background-color: #99c999; } +.birds-eye .test.warn { + background-color: #ffffcc; +} + +.birds-eye .test.warn:hover { + background-color: #f2f28b; +} + .birds-eye .test.fail { background-color: #e9b9b9; } diff --git a/fork-runner/build.gradle b/fork-runner/build.gradle index 4f67e8b6..8d010b3e 100644 --- a/fork-runner/build.gradle +++ b/fork-runner/build.gradle @@ -35,6 +35,9 @@ dependencies { testCompile("junit:junit:$JUNIT_VERSION") { exclude module:'hamcrest-core' } + testCompile( + "org.jmock:jmock:2.8.2", + "org.jmock:jmock-junit4:2.8.2") } jar { diff --git a/fork-runner/src/main/java/com/shazam/fork/Configuration.java b/fork-runner/src/main/java/com/shazam/fork/Configuration.java index 7d4b3aac..75abccce 100644 --- a/fork-runner/src/main/java/com/shazam/fork/Configuration.java +++ b/fork-runner/src/main/java/com/shazam/fork/Configuration.java @@ -30,6 +30,8 @@ public class Configuration { private final String testPackage; private final int testOutputTimeout; private final boolean fallbackToScreenshots; + private final int totalAllowedRetryQuota; + private final int retryPerTestCaseQuota; public Configuration(@Nonnull File androidSdk, @Nonnull File applicationApk, @@ -40,7 +42,9 @@ public Configuration(@Nonnull File androidSdk, @Nonnull Pattern testPackagePattern, @Nonnull String testPackage, int testOutputTimeout, - boolean fallbackToScreenshots) { + boolean fallbackToScreenshots, + int totalAllowedRetryQuota, + int retryPerTestCaseQuota) { this.androidSdk = androidSdk; this.applicationApk = applicationApk; this.instrumentationApk = instrumentationApk; @@ -51,6 +55,8 @@ public Configuration(@Nonnull File androidSdk, this.testPackage = testPackage; this.testOutputTimeout = testOutputTimeout; this.fallbackToScreenshots = fallbackToScreenshots; + this.totalAllowedRetryQuota = totalAllowedRetryQuota; + this.retryPerTestCaseQuota = retryPerTestCaseQuota; } @Nonnull @@ -99,4 +105,12 @@ public int getTestOutputTimeout() { public boolean canFallbackToScreenshots() { return fallbackToScreenshots; } + + public int getTotalAllowedRetryQuota() { + return totalAllowedRetryQuota; + } + + public int getRetryPerTestCaseQuota() { + return retryPerTestCaseQuota; + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java b/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java index 1ce57bd9..1f1c712d 100644 --- a/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java +++ b/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java @@ -11,6 +11,9 @@ import com.shazam.fork.model.InstrumentationInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.util.regex.Pattern; @@ -22,6 +25,8 @@ import static com.shazam.fork.system.axmlparser.InstumentationInfoFactory.parseFromFile; public class ForkBuilder { + private static final Logger logger = LoggerFactory.getLogger(ForkBuilder.class); + private File androidSdk = cleanFile(Defaults.ANDROID_SDK); private File applicationApk; private File instrumentationApk; @@ -30,6 +35,8 @@ public class ForkBuilder { private String testPackage; // Will default to test APK's package name as soon as it's known private int testOutputTimeout = Defaults.TEST_OUTPUT_TIMEOUT_MILLIS; private boolean fallbackToScreenshots = true; + private int totalAllowedRetryQuota = 0; + private int retryPerTestCaseQuota = 1; public static ForkBuilder aFork() { return new ForkBuilder(); @@ -116,6 +123,27 @@ public ForkBuilder withFallbackToScreenshots(boolean fallbackToScreenshots) { return this; } + /** + * Allow to re-run tests. + * + * @param totalAllowedRetryQuota Threshold of total failures below whom the failed tests can be executed again. + * @return this builder + */ + public ForkBuilder withTotalAllowedRetryQuota(int totalAllowedRetryQuota) { + this.totalAllowedRetryQuota = totalAllowedRetryQuota; + return this; + } + + /** + * Max number of time each testCase is attempted again. + * @param retryPerTestCaseQuota max number of attempts when rerunning tests. + * @return this builder. + */ + public ForkBuilder withRetryPerTestCaseQuota(int retryPerTestCaseQuota) { + this.retryPerTestCaseQuota = retryPerTestCaseQuota; + return this; + } + public Fork build() { checkNotNull(androidSdk, "SDK is required."); checkArgument(androidSdk.exists(), "SDK directory does not exist."); @@ -125,6 +153,9 @@ public Fork build() { checkArgument(instrumentationApk.exists(), "Instrumentation APK file does not exist."); checkNotNull(output, "Output path is required."); checkArgument(testOutputTimeout >= 0, "Timeout must be non-negative."); + checkArgument(totalAllowedRetryQuota >= 0, "Total allowed retry quota should not be negative."); + checkArgument(retryPerTestCaseQuota >= 0, "Retry per test case quota should not be negative."); + logArgumentsBadInteractions(); InstrumentationInfo instrumentationInfo = parseFromFile(instrumentationApk); String testPackage = configuredOrInstrumentationPackage(instrumentationInfo.getInstrumentationPackage()); @@ -138,11 +169,20 @@ public Fork build() { compilePatternFor(testPackage), testPackage, testOutputTimeout, - fallbackToScreenshots - ); + fallbackToScreenshots, + totalAllowedRetryQuota, + retryPerTestCaseQuota); return new Fork(configuration); } + private void logArgumentsBadInteractions() { + if(totalAllowedRetryQuota > 0 && totalAllowedRetryQuota < retryPerTestCaseQuota){ + logger.warn("Total allowed retry quota ["+totalAllowedRetryQuota+"] " + + "is smaller than Retry per test case quota ["+retryPerTestCaseQuota+"]. " + + "This is suspicious as the fist mentioned parameter is an overall cap."); + } + } + private String configuredOrInstrumentationPackage(String instrumentationPackage) { if (testPackage != null) { return testPackage; diff --git a/fork-runner/src/main/java/com/shazam/fork/ForkCli.java b/fork-runner/src/main/java/com/shazam/fork/ForkCli.java index 3a26bd79..98087d46 100644 --- a/fork-runner/src/main/java/com/shazam/fork/ForkCli.java +++ b/fork-runner/src/main/java/com/shazam/fork/ForkCli.java @@ -63,6 +63,14 @@ public static class CommandLineArgs { @Parameter(names = { "-h", "--help" }, description = "Command help", help = true, hidden = true) public boolean help; + + @Parameter(names = { "--total-allowed-retry-quota" }, description = "Amount of re-executions of failing tests allowed.", converter = IntegerConverter.class) + public int totalAllowedRetryQuota = 0; + + @Parameter(names = { "--retry-per-test-case-quota" }, description = "Max number of time each testCase is attempted again " + + "before declaring it as a failure.", converter = IntegerConverter.class) + public int retryPerTestCaseQuota = 1; + } /* JCommander deems it necessary that this class be public. Lame. */ @@ -125,5 +133,13 @@ private static void overrideDefaultsIfSet(ForkBuilder forkBuilder, CommandLineAr if (parsedArgs.testOutputTimeout > -1) { forkBuilder.withTestOutputTimeout(parsedArgs.testOutputTimeout); } + + if (parsedArgs.totalAllowedRetryQuota > 0) { + forkBuilder.withTotalAllowedRetryQuota(parsedArgs.totalAllowedRetryQuota); + } + + if(parsedArgs.retryPerTestCaseQuota > -1){ + forkBuilder.withRetryPerTestCaseQuota(parsedArgs.retryPerTestCaseQuota); + } } } diff --git a/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java b/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java index 7a8be308..b472abcf 100755 --- a/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java @@ -13,10 +13,12 @@ package com.shazam.fork; import com.shazam.fork.model.Pool; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.pooling.NoDevicesForPoolException; import com.shazam.fork.pooling.PoolLoader; -import com.shazam.fork.runner.*; +import com.shazam.fork.runner.PoolTestRunner; +import com.shazam.fork.runner.PoolTestRunnerFactory; +import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.suite.TestClassLoader; import com.shazam.fork.suite.TestClassScanningException; import com.shazam.fork.summary.SummaryGeneratorHook; @@ -60,12 +62,12 @@ public boolean run() { CountDownLatch poolCountDownLatch = new CountDownLatch(numberOfPools); poolExecutor = namedExecutor(numberOfPools, "PoolExecutor-%d"); - List testClasses = testClassLoader.loadTestClasses(); - summaryGeneratorHook.registerHook(pools, testClasses); + List testCases = testClassLoader.loadTestClasses(); + summaryGeneratorHook.registerHook(pools, testCases); progressReporter.start(); for (Pool pool : pools) { - PoolTestRunner poolTestRunner = poolTestRunnerFactory.createPoolTestRunner(pool, testClasses, + PoolTestRunner poolTestRunner = poolTestRunnerFactory.createPoolTestRunner(pool, testCases, poolCountDownLatch, progressReporter); poolExecutor.execute(poolTestRunner); } diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java index 7467a9ee..8c1781b2 100644 --- a/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java +++ b/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java @@ -17,12 +17,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.shazam.fork.utils.Utils.millisSinceNanoTime; -import static com.shazam.fork.injector.runner.ProgressReporterInjector.progressReporter; import static com.shazam.fork.injector.pooling.PoolLoaderInjector.poolLoader; import static com.shazam.fork.injector.runner.PoolTestRunnerFactoryInjector.poolTestRunnerFactory; +import static com.shazam.fork.injector.runner.ProgressReporterInjector.progressReporter; import static com.shazam.fork.injector.suite.TestClassLoaderInjector.testClassLoader; import static com.shazam.fork.injector.summary.SummaryGeneratorHookInjector.summaryGeneratorHook; +import static com.shazam.fork.utils.Utils.millisSinceNanoTime; import static java.lang.System.nanoTime; public class ForkRunnerInjector { diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/accumulator/PoolTestCaseFailureAccumulatorInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/accumulator/PoolTestCaseFailureAccumulatorInjector.java new file mode 100644 index 00000000..2f84d6e3 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/injector/accumulator/PoolTestCaseFailureAccumulatorInjector.java @@ -0,0 +1,9 @@ +package com.shazam.fork.injector.accumulator; + +import com.shazam.fork.model.PoolTestCaseFailureAccumulator; + +public class PoolTestCaseFailureAccumulatorInjector { + public static PoolTestCaseFailureAccumulator poolTestCaseFailureAccumulator() { + return new PoolTestCaseFailureAccumulator(); + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/runner/PoolProgressTrackersInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/runner/PoolProgressTrackersInjector.java new file mode 100644 index 00000000..e239be6f --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/injector/runner/PoolProgressTrackersInjector.java @@ -0,0 +1,14 @@ +package com.shazam.fork.injector.runner; + +import com.shazam.fork.model.Pool; +import com.shazam.fork.runner.PoolProgressTracker; + +import java.util.Map; + +import static com.beust.jcommander.internal.Maps.newHashMap; + +public class PoolProgressTrackersInjector { + public static Map poolProgressTrackers() { + return newHashMap(); + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/runner/PoolTestRunnerFactoryInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/runner/PoolTestRunnerFactoryInjector.java index 4d8a2fae..7c621be6 100644 --- a/fork-runner/src/main/java/com/shazam/fork/injector/runner/PoolTestRunnerFactoryInjector.java +++ b/fork-runner/src/main/java/com/shazam/fork/injector/runner/PoolTestRunnerFactoryInjector.java @@ -12,13 +12,11 @@ import com.shazam.fork.runner.PoolTestRunnerFactory; -import static com.shazam.fork.injector.ConfigurationInjector.configuration; import static com.shazam.fork.injector.runner.DeviceTestRunnerFactoryInjector.deviceTestRunnerFactory; -import static com.shazam.fork.injector.system.FileManagerInjector.fileManager; public class PoolTestRunnerFactoryInjector { public static PoolTestRunnerFactory poolTestRunnerFactory() { - return new PoolTestRunnerFactory(configuration(), fileManager(), deviceTestRunnerFactory()); + return new PoolTestRunnerFactory(deviceTestRunnerFactory()); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java index 155e1913..35cb24d0 100644 --- a/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java +++ b/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java @@ -10,12 +10,18 @@ package com.shazam.fork.injector.runner; -import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.runner.OverallProgressReporter; +import com.shazam.fork.runner.ProgressReporter; + +import static com.shazam.fork.injector.ConfigurationInjector.configuration; +import static com.shazam.fork.injector.runner.PoolProgressTrackersInjector.poolProgressTrackers; +import static com.shazam.fork.injector.accumulator.PoolTestCaseFailureAccumulatorInjector.poolTestCaseFailureAccumulator; public class ProgressReporterInjector { public static ProgressReporter progressReporter() { - return new OverallProgressReporter(); + return new OverallProgressReporter(configuration(), + poolProgressTrackers(), + poolTestCaseFailureAccumulator()); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/model/PoolTestCaseAccumulator.java b/fork-runner/src/main/java/com/shazam/fork/model/PoolTestCaseAccumulator.java new file mode 100644 index 00000000..06175157 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/PoolTestCaseAccumulator.java @@ -0,0 +1,9 @@ +package com.shazam.fork.model; + +public interface PoolTestCaseAccumulator { + void record(Pool pool, TestCaseEvent testCaseEvent); + + int getCount(Pool pool, TestCaseEvent testCaseEvent); + + int getCount(TestCaseEvent testCaseEvent); +} diff --git a/fork-runner/src/main/java/com/shazam/fork/model/PoolTestCaseFailureAccumulator.java b/fork-runner/src/main/java/com/shazam/fork/model/PoolTestCaseFailureAccumulator.java new file mode 100644 index 00000000..8966b0d3 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/PoolTestCaseFailureAccumulator.java @@ -0,0 +1,69 @@ +package com.shazam.fork.model; + +import com.google.common.base.Predicate; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.SetMultimap; + +import static com.google.common.collect.FluentIterable.from; + +/** + * Class that keeps track of the number of times each testCase is executed for device. + */ +public class PoolTestCaseFailureAccumulator implements PoolTestCaseAccumulator { + + private SetMultimap map = HashMultimap.create(); + + @Override + public void record(Pool pool, TestCaseEvent testCaseEvent) { + if (!map.containsKey(pool)) { + map.put(pool, createNew(testCaseEvent)); + } + + if (!from(map.get(pool)).anyMatch(isSameTestCase(testCaseEvent))) { + map.get(pool).add( + createNew(testCaseEvent) + .withIncreasedCount()); + } else { + from(map.get(pool)) + .firstMatch(isSameTestCase(testCaseEvent)).get() + .increaseCount(); + } + } + + @Override + public int getCount(Pool pool, TestCaseEvent testCaseEvent) { + if (map.containsKey(pool)) { + return from(map.get(pool)) + .firstMatch(isSameTestCase(testCaseEvent)).or(TestCaseEventCounter.EMPTY) + .getCount(); + } else { + return 0; + } + } + + @Override + public int getCount(TestCaseEvent testCaseEvent) { + int result = 0; + ImmutableList counters = from(map.values()) + .filter(isSameTestCase(testCaseEvent)).toList(); + for (TestCaseEventCounter counter : counters) { + result += counter.getCount(); + } + return result; + } + + private static TestCaseEventCounter createNew(final TestCaseEvent testCaseEvent) { + return new TestCaseEventCounter(testCaseEvent, 0); + } + + private static Predicate isSameTestCase(final TestCaseEvent testCaseEvent) { + return new Predicate() { + @Override + public boolean apply(TestCaseEventCounter input) { + return input.getTestCaseEvent() != null + && testCaseEvent.equals(input.getTestCaseEvent()); + } + }; + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java new file mode 100644 index 00000000..743dff94 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java @@ -0,0 +1,55 @@ +package com.shazam.fork.model; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.google.common.base.Objects; + +import javax.annotation.Nonnull; + +public class TestCaseEvent { + + private final String testMethod; + private final String testClass; + private final boolean isIgnored; + + public TestCaseEvent(String testMethod, String testClass, boolean isIgnored) { + this.testMethod = testMethod; + this.testClass = testClass; + this.isIgnored = isIgnored; + } + + public static TestCaseEvent newTestCase(String testMethod, String testClass, boolean isIgnored){ + return new TestCaseEvent(testMethod, testClass, isIgnored); + } + + public static TestCaseEvent newTestCase(@Nonnull TestIdentifier testIdentifier, boolean isIgnored){ + return new TestCaseEvent(testIdentifier.getTestName(), testIdentifier.getClassName(), isIgnored); + } + + public String getTestMethod() { + return testMethod; + } + + public String getTestClass() { + return testClass; + } + + public boolean isIgnored() { + return isIgnored; + } + + @Override + public int hashCode() { + return Objects.hashCode( this.testMethod, this.testClass); + } + + @Override + public boolean equals(Object obj) { + + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + final TestCaseEvent other = (TestCaseEvent) obj; + return Objects.equal(this.testMethod, other.testMethod) + && Objects.equal(this.testClass, other.testClass); + } + +} diff --git a/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEventCounter.java b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEventCounter.java new file mode 100644 index 00000000..7b4c3af2 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEventCounter.java @@ -0,0 +1,48 @@ +package com.shazam.fork.model; + +import com.google.common.base.Objects; + +import java.util.concurrent.atomic.AtomicInteger; + +public class TestCaseEventCounter { + + public static final TestCaseEventCounter EMPTY = new TestCaseEventCounter(null, 0); + + private TestCaseEvent testCaseEvent; + private AtomicInteger count; + + public TestCaseEventCounter(TestCaseEvent testCaseEvent, int initialCount) { + this.testCaseEvent = testCaseEvent; + this.count = new AtomicInteger(initialCount); + } + + public int increaseCount() { + return count.incrementAndGet(); + } + + public TestCaseEvent getTestCaseEvent() { + return testCaseEvent; + } + + public int getCount() { + return count.get(); + } + + public TestCaseEventCounter withIncreasedCount() { + increaseCount(); + return this; + } + + @Override + public int hashCode() { + return testCaseEvent.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + final TestCaseEventCounter other = (TestCaseEventCounter) obj; + return Objects.equal(this.testCaseEvent, other.testCaseEvent); + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java index 804ea492..a6940152 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java @@ -13,7 +13,9 @@ package com.shazam.fork.runner; import com.android.ddmlib.IDevice; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.system.adb.Installer; import org.slf4j.Logger; @@ -31,7 +33,7 @@ public class DeviceTestRunner implements Runnable { private final Installer installer; private final Pool pool; private final Device device; - private final Queue queueOfTestsInPool; + private final Queue queueOfTestsInPool; private final CountDownLatch deviceCountDownLatch; private final ProgressReporter progressReporter; private final TestRunFactory testRunFactory; @@ -39,21 +41,21 @@ public class DeviceTestRunner implements Runnable { public DeviceTestRunner(Installer installer, Pool pool, Device device, - Queue queueOfTestsInPool, + Queue queueOfTestsInPool, CountDownLatch deviceCountDownLatch, ProgressReporter progressReporter, TestRunFactory testRunFactory) { this.installer = installer; - this.pool = pool; - this.device = device; + this.pool = pool; + this.device = device; this.queueOfTestsInPool = queueOfTestsInPool; this.deviceCountDownLatch = deviceCountDownLatch; this.progressReporter = progressReporter; this.testRunFactory = testRunFactory; } - @Override - public void run() { + @Override + public void run() { IDevice deviceInterface = device.getDeviceInterface(); try { installer.prepareInstallation(deviceInterface); @@ -61,14 +63,18 @@ public void run() { removeRemoteDirectory(deviceInterface); createRemoteDirectory(deviceInterface); - TestClass testClass; - while ((testClass = queueOfTestsInPool.poll()) != null) { - TestRun testRun = testRunFactory.createTestRun(testClass, device, pool, progressReporter); + TestCaseEvent testCaseEvent; + while ((testCaseEvent = queueOfTestsInPool.poll()) != null) { + TestRun testRun = testRunFactory.createTestRun(testCaseEvent, + device, + pool, + progressReporter, + queueOfTestsInPool); testRun.execute(); } - } finally { + } finally { logger.info("Device {} from pool {} finished", device.getSerial(), pool.getName()); - deviceCountDownLatch.countDown(); - } - } + deviceCountDownLatch.countDown(); + } + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java index 1ad57676..14c7d18c 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java @@ -27,10 +27,11 @@ public DeviceTestRunnerFactory(Installer installer, TestRunFactory testRunFactor } public Runnable createDeviceTestRunner(Pool pool, - Queue testClassQueue, + Queue testClassQueue, CountDownLatch deviceInPoolCountDownLatch, Device device, - ProgressReporter progressReporter) { + ProgressReporter progressReporter + ) { return new DeviceTestRunner( installer, pool, diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java index 318273cf..14e6376b 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java @@ -10,10 +10,16 @@ package com.shazam.fork.runner; +import com.shazam.fork.Configuration; import com.shazam.fork.model.Pool; +import com.shazam.fork.model.PoolTestCaseAccumulator; +import com.shazam.fork.model.TestCaseEvent; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import static com.shazam.fork.utils.Utils.millisBetweenNanoTimes; import static com.shazam.fork.utils.Utils.millisSinceNanoTime; @@ -23,9 +29,20 @@ * Reports for stats on all the pools. */ public class OverallProgressReporter implements ProgressReporter { + private long startOfTests; private long endOfTests; - private final Map poolProgressTrackers = new HashMap<>(); + private final Map poolProgressTrackers; + private final RetryWatchdog retryWatchdog; + private final PoolTestCaseAccumulator failedTestCasesAccumulator; + + public OverallProgressReporter(Configuration configuration, + Map poolProgressTrackers, + PoolTestCaseAccumulator failedTestCasesAccumulator) { + this.retryWatchdog = new RetryWatchdog(configuration); + this.poolProgressTrackers = poolProgressTrackers; + this.failedTestCasesAccumulator = failedTestCasesAccumulator; + } @Override public void start() { @@ -76,4 +93,59 @@ public float getProgress() { return progress / size; } + public boolean requestRetry(Pool pool, TestCaseEvent testCase) { + boolean result = retryWatchdog.requestRetry(failedTestCasesAccumulator.getCount(testCase)); + if (result && poolProgressTrackers.containsKey(pool)) { + poolProgressTrackers.get(pool).trackTestEnqueuedAgain(); + } + return result; + } + + @Override + public void recordFailedTestCase(Pool pool, TestCaseEvent testCase) { + failedTestCasesAccumulator.record(pool, testCase); + } + + @Override + public int getTestFailuresCount(Pool pool, TestCaseEvent testCase) { + return failedTestCasesAccumulator.getCount(pool, testCase); + } + + private class RetryWatchdog { + private final Logger logger = LoggerFactory.getLogger(RetryWatchdog.class); + + private final int maxRetryPerTestCaseQuota; + private AtomicInteger totalAllowedRetryLeft; + private StringBuilder logBuilder = new StringBuilder(); + + public RetryWatchdog(Configuration configuration) { + totalAllowedRetryLeft = new AtomicInteger(configuration.getTotalAllowedRetryQuota()); + maxRetryPerTestCaseQuota = configuration.getRetryPerTestCaseQuota(); + } + + public boolean requestRetry(int currentSingleTestCaseFailures) { + boolean totalAllowedRetryAvailable = totalAllowedRetryAvailable(); + boolean singleTestAllowed = currentSingleTestCaseFailures <= maxRetryPerTestCaseQuota; + boolean result = totalAllowedRetryAvailable && singleTestAllowed; + + log(currentSingleTestCaseFailures, singleTestAllowed, result); + return result; + } + + private boolean totalAllowedRetryAvailable() { + return totalAllowedRetryLeft.get() > 0 && totalAllowedRetryLeft.getAndDecrement() >= 0; + } + + private void log(int testCaseFailures, boolean singleTestAllowed, boolean result) { + if(logger.isDebugEnabled()) { + logBuilder.setLength(0); //clean up. + logBuilder.append("Retry requested ") + .append(result ? " and allowed. " : " but not allowed. ") + .append("Total retry left :").append(totalAllowedRetryLeft.get()) + .append(" and Single Test case retry left: ") + .append(singleTestAllowed ? maxRetryPerTestCaseQuota - testCaseFailures : 0); + logger.debug(logBuilder.toString()); + } + } + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTracker.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTracker.java index 03bdafd6..9d10ba69 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTracker.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTracker.java @@ -10,29 +10,14 @@ package com.shazam.fork.runner; -public class PoolProgressTracker { +public interface PoolProgressTracker { + void completedTest(); - private final int totalTests; - private int failedTests; - private int completedTests; + void failedTest(); - public PoolProgressTracker(int totalTests) { - this.totalTests = totalTests; - } + void trackTestEnqueuedAgain(); - public void completedTest() { - completedTests++; - } + float getProgress(); - public void failedTest() { - failedTests++; - } - - public float getProgress() { - return (float) completedTests / (float) totalTests; - } - - public int getNumberOfFailedTests() { - return failedTests; - } + int getNumberOfFailedTests(); } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTrackerImpl.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTrackerImpl.java new file mode 100644 index 00000000..a4725207 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolProgressTrackerImpl.java @@ -0,0 +1,48 @@ +/* + * Copyright 2015 Shazam Entertainment Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.shazam.fork.runner; + +public class PoolProgressTrackerImpl implements PoolProgressTracker { + + private final int totalTests; + private int failedTests; + private int completedTests; + + public PoolProgressTrackerImpl(int totalTests) { + this.totalTests = totalTests; + } + + @Override + public void completedTest() { + completedTests++; + } + + @Override + public void failedTest() { + failedTests++; + } + + @Override + public void trackTestEnqueuedAgain() { + completedTests--; + failedTests--; + } + + @Override + public float getProgress() { + return (float) completedTests / (float) totalTests; + } + + @Override + public int getNumberOfFailedTests() { + return failedTests; + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java index 5df6ed3d..c44db1e4 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java @@ -12,45 +12,35 @@ */ package com.shazam.fork.runner; -import com.android.ddmlib.testrunner.TestIdentifier; -import com.shazam.fork.Configuration; -import com.shazam.fork.model.*; -import com.shazam.fork.runner.listeners.ForkXmlTestRunListener; -import com.shazam.fork.system.io.FileManager; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.Queue; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import static com.shazam.fork.Utils.namedExecutor; -import static com.shazam.fork.model.Device.Builder.aDevice; -import static com.shazam.fork.runner.listeners.TestRunListenersFactory.getForkXmlTestRunListener; public class PoolTestRunner implements Runnable { private final Logger logger = LoggerFactory.getLogger(PoolTestRunner.class); public static final String DROPPED_BY = "DroppedBy-"; - private final Configuration configuration; - private final FileManager fileManager; private final Pool pool; - private final Queue testClasses; + private final Queue testCases; private final CountDownLatch poolCountDownLatch; private final DeviceTestRunnerFactory deviceTestRunnerFactory; private final ProgressReporter progressReporter; - public PoolTestRunner(Configuration configuration, - FileManager fileManager, - DeviceTestRunnerFactory deviceTestRunnerFactory, Pool pool, - Queue testClasses, + public PoolTestRunner(DeviceTestRunnerFactory deviceTestRunnerFactory, Pool pool, + Queue testCases, CountDownLatch poolCountDownLatch, ProgressReporter progressReporter) { - this.configuration = configuration; - this.fileManager = fileManager; this.pool = pool; - this.testClasses = testClasses; + this.testCases = testCases; this.poolCountDownLatch = poolCountDownLatch; this.deviceTestRunnerFactory = deviceTestRunnerFactory; this.progressReporter = progressReporter; @@ -65,7 +55,7 @@ public void run() { CountDownLatch deviceCountDownLatch = new CountDownLatch(devicesInPool); logger.info("Pool {} started", poolName); for (Device device : pool.getDevices()) { - Runnable deviceTestRunner = deviceTestRunnerFactory.createDeviceTestRunner(pool, testClasses, + Runnable deviceTestRunner = deviceTestRunnerFactory.createDeviceTestRunner(pool, testCases, deviceCountDownLatch, device, progressReporter); concurrentDeviceExecutor.execute(deviceTestRunner); } @@ -73,7 +63,6 @@ public void run() { } catch (InterruptedException e) { logger.warn("Pool {} was interrupted while running", poolName); } finally { - failAnyDroppedClasses(pool, testClasses); if (concurrentDeviceExecutor != null) { concurrentDeviceExecutor.shutdown(); } @@ -82,34 +71,4 @@ public void run() { logger.info("Pools remaining: {}", poolCountDownLatch.getCount()); } } - - /** - * Only generate XML files for dropped classes the console listener and logcat listeners aren't relevant to - * dropped tests. - *

- * In particular, not triggering the console listener will probably make the flaky report better. - */ - private void failAnyDroppedClasses(Pool pool, Queue testClassQueue) { - HashMap emptyHash = new HashMap<>(); - TestClass nextTest; - while ((nextTest = testClassQueue.poll()) != null) { - String className = nextTest.getName(); - String poolName = pool.getName(); - Device failedTestsDevice = aDevice().withSerial(DROPPED_BY + poolName).build(); - ForkXmlTestRunListener xmlGenerator = getForkXmlTestRunListener(fileManager, configuration.getOutput(), - pool, failedTestsDevice, nextTest); - - Collection methods = nextTest.getUnignoredMethods(); - xmlGenerator.testRunStarted(poolName, methods.size()); - for (TestMethod method : methods) { - String methodName = method.getName(); - TestIdentifier identifier = new TestIdentifier(className, methodName); - xmlGenerator.testStarted(identifier); - xmlGenerator.testFailed(identifier, poolName + " DROPPED"); - xmlGenerator.testEnded(identifier, emptyHash); - } - xmlGenerator.testRunFailed("DROPPED BY " + poolName); - xmlGenerator.testRunEnded(0, emptyHash); - } - } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java index fbfbe30d..b89c057a 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java @@ -10,51 +10,33 @@ package com.shazam.fork.runner; -import com.shazam.fork.Configuration; import com.shazam.fork.model.Pool; -import com.shazam.fork.model.TestClass; -import com.shazam.fork.system.io.FileManager; +import com.shazam.fork.model.TestCaseEvent; import java.util.LinkedList; import java.util.List; import java.util.concurrent.CountDownLatch; public class PoolTestRunnerFactory { - private final Configuration configuration; - private final FileManager fileManager; private final DeviceTestRunnerFactory deviceTestRunnerFactory; - public PoolTestRunnerFactory(Configuration configuration, - FileManager fileManager, - DeviceTestRunnerFactory deviceTestRunnerFactory) { - this.configuration = configuration; - this.fileManager = fileManager; + public PoolTestRunnerFactory(DeviceTestRunnerFactory deviceTestRunnerFactory) { this.deviceTestRunnerFactory = deviceTestRunnerFactory; } public PoolTestRunner createPoolTestRunner(Pool pool, - List testClasses, + List testCases, CountDownLatch poolCountDownLatch, ProgressReporter progressReporter) { - int totalTests = countTests(testClasses); - progressReporter.addPoolProgress(pool, new PoolProgressTracker(totalTests)); + int totalTests = testCases.size(); + progressReporter.addPoolProgress(pool, new PoolProgressTrackerImpl(totalTests)); return new PoolTestRunner( - configuration, - fileManager, deviceTestRunnerFactory, pool, - new LinkedList<>(testClasses), + new LinkedList<>(testCases), poolCountDownLatch, progressReporter); } - - private int countTests(List testClasses) { - int sum = 0; - for (TestClass testClass : testClasses) { - sum += testClass.getMethods().size(); - } - return sum; - } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java index a2200ced..a99113c8 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java @@ -11,6 +11,7 @@ package com.shazam.fork.runner; import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; public interface ProgressReporter { @@ -32,4 +33,10 @@ public interface ProgressReporter { int getFailures(); float getProgress(); + + boolean requestRetry(Pool pool, TestCaseEvent testCaseEvent); + + void recordFailedTestCase(Pool pool, TestCaseEvent testCase); + + int getTestFailuresCount(Pool pool, TestCaseEvent testCase); } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java index 6467dca5..f19e3c80 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java @@ -12,9 +12,13 @@ */ package com.shazam.fork.runner; -import com.android.ddmlib.*; -import com.android.ddmlib.testrunner.*; -import com.shazam.fork.model.TestClass; +import com.android.ddmlib.AdbCommandRejectedException; +import com.android.ddmlib.ShellCommandUnresponsiveException; +import com.android.ddmlib.TimeoutException; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.ITestRunListener; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.shazam.fork.model.TestCaseEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,14 +48,15 @@ public void execute() { testRunParameters.getTestRunner(), testRunParameters.getDeviceInterface()); - TestClass test = testRunParameters.getTest(); - String testClassName = test.getName(); + TestCaseEvent test = testRunParameters.getTest(); + String testClassName = test.getTestClass(); + String testMethodName = test.getTestMethod(); IRemoteAndroidTestRunner.TestSize testSize = testRunParameters.getTestSize(); if (testSize != null) { runner.setTestSize(testSize); } runner.setRunName(poolName); - runner.setClassName(testClassName); + runner.setMethodName(testClassName, testMethodName); runner.setMaxtimeToOutputResponse(testRunParameters.getTestOutputTimeout()); try { runner.run(testRunListeners); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java index c90118de..87c4d028 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java @@ -13,10 +13,14 @@ import com.android.ddmlib.testrunner.ITestRunListener; import com.shazam.fork.Configuration; import com.shazam.fork.RuntimeConfiguration; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.InstrumentationInfo; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.runner.listeners.TestRunListenersFactory; import java.util.List; +import java.util.Queue; import static com.shazam.fork.runner.TestRunParameters.Builder.testRunParameters; @@ -34,12 +38,16 @@ public TestRunFactory(Configuration configuration, this.testRunListenersFactory = testRunListenersFactory; } - public TestRun createTestRun(TestClass testClass, Device device, Pool pool, ProgressReporter progressReporter) { + public TestRun createTestRun(TestCaseEvent testCase, + Device device, + Pool pool, + ProgressReporter progressReporter, + Queue queueOfTestsInPool ) { InstrumentationInfo instrumentationInfo = configuration.getInstrumentationInfo(); TestRunParameters testRunParameters = testRunParameters() .withDeviceInterface(device.getDeviceInterface()) - .withTest(testClass) + .withTest(testCase) .withTestPackage(instrumentationInfo.getInstrumentationPackage()) .withTestRunner(instrumentationInfo.getTestRunnerClass()) .withTestSize(runtimeConfiguration.getTestSize()) @@ -47,10 +55,11 @@ public TestRun createTestRun(TestClass testClass, Device device, Pool pool, Prog .build(); List testRunListeners = testRunListenersFactory.createTestListeners( - testClass, + testCase, device, pool, - progressReporter); + progressReporter, + queueOfTestsInPool); return new TestRun( pool.getName(), diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java index dcd599a0..6a236d13 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java @@ -14,19 +14,19 @@ import com.android.ddmlib.IDevice; import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import javax.annotation.Nullable; public class TestRunParameters { - private final TestClass test; + private final TestCaseEvent test; private final String testPackage; private final String testRunner; private final IRemoteAndroidTestRunner.TestSize testSize; private final int testOutputTimeout; private final IDevice deviceInterface; - public TestClass getTest() { + public TestCaseEvent getTest() { return test; } @@ -52,7 +52,7 @@ public IDevice getDeviceInterface() { } public static class Builder { - private TestClass test; + private TestCaseEvent test; private String testPackage; private String testRunner; private IRemoteAndroidTestRunner.TestSize testSize; @@ -63,7 +63,7 @@ public static Builder testRunParameters() { return new Builder(); } - public Builder withTest(TestClass test) { + public Builder withTest(TestCaseEvent test) { this.test = test; return this; } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java index 67a7f96d..01fb1def 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java @@ -12,27 +12,71 @@ */ package com.shazam.fork.runner.listeners; +import com.android.ddmlib.testrunner.TestIdentifier; import com.android.ddmlib.testrunner.XmlTestRunListener; -import com.shazam.fork.model.*; +import com.google.common.collect.ImmutableMap; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; +import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.system.io.FileManager; +import com.shazam.fork.system.io.FileType; import java.io.File; +import java.util.Map; + +import javax.annotation.Nonnull; + +import static com.shazam.fork.summary.TestResult.SUMMARY_KEY_TOTAL_FAILURE_COUNT; public class ForkXmlTestRunListener extends XmlTestRunListener { + private final FileManager fileManager; private final Pool pool; private final Device device; - private final TestClass testClass; + private final TestCaseEvent testCase; - public ForkXmlTestRunListener(FileManager fileManager, Pool pool, Device device, TestClass testClass) { + @Nonnull + private + ProgressReporter progressReporter; + TestIdentifier test; + + public ForkXmlTestRunListener(FileManager fileManager, + Pool pool, + Device device, + TestCaseEvent testCase, + @Nonnull ProgressReporter progressReporter) { this.fileManager = fileManager; this.pool = pool; this.device = device; - this.testClass = testClass; + this.testCase = testCase; + this.progressReporter = progressReporter; } @Override protected File getResultFile(File reportDir) { - return fileManager.createFileForTest(pool, device, testClass); + return fileManager.createFile(FileType.TEST, pool, device, testCase); + } + + @Override + public void testStarted(TestIdentifier test) { + this.test = test; + super.testStarted(test); + } + + @Override + protected Map getPropertiesAttributes() { + + ImmutableMap.Builder mapBuilder = ImmutableMap.builder() + .putAll(super.getPropertiesAttributes()); + if (test != null) { + int testFailuresCount = progressReporter.getTestFailuresCount(pool, new TestCaseEvent(test.getTestName(), test.getClassName(), false)); + if (testFailuresCount > 0) { + mapBuilder + .put(SUMMARY_KEY_TOTAL_FAILURE_COUNT, Integer.toString(testFailuresCount)) + .build(); + } + } + return mapBuilder.build(); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java new file mode 100644 index 00000000..294430c7 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java @@ -0,0 +1,93 @@ +/* + * Copyright 2016 Shazam Entertainment Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.shazam.fork.runner.listeners; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; +import com.shazam.fork.runner.ProgressReporter; +import com.shazam.fork.system.io.FileManager; +import com.shazam.fork.system.io.FileType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Map; +import java.util.Queue; + +import javax.annotation.Nonnull; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.shazam.fork.model.TestCaseEvent.newTestCase; + +public class RetryListener extends NoOpITestRunListener { + + private static final Logger logger = LoggerFactory.getLogger(RetryListener.class); + + @Nonnull + private final Device device; + private TestIdentifier failedTest; + @Nonnull + private final Queue queueOfTestsInPool; + @Nonnull + private final TestCaseEvent currentTestCaseEvent; + private ProgressReporter progressReporter; + private FileManager fileManager; + private Pool pool; + + public RetryListener(@Nonnull Pool pool, @Nonnull Device device, + @Nonnull Queue queueOfTestsInPool, + @Nonnull TestCaseEvent currentTestCaseEvent, + @Nonnull ProgressReporter progressReporter, + FileManager fileManager) { + checkNotNull(device); + checkNotNull(queueOfTestsInPool); + checkNotNull(currentTestCaseEvent); + checkNotNull(progressReporter); + checkNotNull(pool); + this.device = device; + this.queueOfTestsInPool = queueOfTestsInPool; + this.currentTestCaseEvent = currentTestCaseEvent; + this.progressReporter = progressReporter; + this.pool = pool; + this.fileManager = fileManager; + } + + @Override + public void testFailed(TestIdentifier test, String trace) { + failedTest = test; + progressReporter.recordFailedTestCase(pool, newTestCase(failedTest, false)); + } + + @Override + public void testRunEnded(long elapsedTime, Map runMetrics) { + super.testRunEnded(elapsedTime, runMetrics); + if (failedTest != null) { + if (progressReporter.requestRetry(pool, newTestCase(failedTest, false))) { + queueOfTestsInPool.add(currentTestCaseEvent); + logger.info("Test " + failedTest.toString() + " enqueued again into pool:" + pool.getName()); + removeFailureTraceFiles(); + } else { + logger.info("Test " + failedTest.toString() + " failed on device " + device.getSafeSerial() + " but retry is not allowed."); + } + } + } + + public void removeFailureTraceFiles( ) { + final File file = fileManager.getFile(FileType.TEST, pool.getName(), device.getSafeSerial(), failedTest); + boolean deleted = file.delete(); + if(!deleted){ + logger.warn("Failed to remove file " + file.getAbsoluteFile() + " for a failed but enqueued again test"); + } + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java index fea49059..72b093fd 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java @@ -13,12 +13,15 @@ import com.android.ddmlib.testrunner.ITestRunListener; import com.google.gson.Gson; import com.shazam.fork.Configuration; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.system.io.FileManager; import java.io.File; import java.util.List; +import java.util.Queue; import static com.shazam.fork.model.Diagnostics.SCREENSHOTS; import static com.shazam.fork.model.Diagnostics.VIDEO; @@ -38,26 +41,30 @@ public TestRunListenersFactory(Configuration configuration, this.gson = gson; } - public List createTestListeners(TestClass testClass, + public List createTestListeners(TestCaseEvent testCase, Device device, Pool pool, - ProgressReporter progressReporter) { + ProgressReporter progressReporter, + Queue testCaseEventQueue) { return asList( new ProgressTestRunListener(pool, progressReporter), - getForkXmlTestRunListener(fileManager, configuration.getOutput(), pool, device, testClass), + getForkXmlTestRunListener(fileManager, configuration.getOutput(), pool, device, testCase, progressReporter), new ConsoleLoggingTestRunListener(configuration.getTestPackage(), device.getSerial(), device.getModelName(), progressReporter), new LogCatTestRunListener(gson, fileManager, pool, device), new SlowWarningTestRunListener(), - getScreenTraceTestRunListener(fileManager, pool, device)); + getScreenTraceTestRunListener(fileManager, pool, device), + new RetryListener(pool, device, testCaseEventQueue, testCase, progressReporter, fileManager)); } - public static ForkXmlTestRunListener getForkXmlTestRunListener(FileManager fileManager, + + private ForkXmlTestRunListener getForkXmlTestRunListener(FileManager fileManager, File output, Pool pool, Device device, - TestClass testClass) { - ForkXmlTestRunListener xmlTestRunListener = new ForkXmlTestRunListener(fileManager, pool, device, testClass); + TestCaseEvent testCase, + ProgressReporter progressReporter) { + ForkXmlTestRunListener xmlTestRunListener = new ForkXmlTestRunListener(fileManager, pool, device, testCase, progressReporter); xmlTestRunListener.setReportDir(output); return xmlTestRunListener; } diff --git a/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java b/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java index 1e07da88..e71ab06b 100644 --- a/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java +++ b/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java @@ -10,21 +10,56 @@ package com.shazam.fork.suite; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestMethod; +import java.util.Collections; import java.util.List; +import javax.annotation.Nullable; + +import static com.google.common.collect.FluentIterable.from; + public class TestClassLoader { private final TestClassScanner scanner; private final TestClassFilter filter; + private Predicate VALID_METHOD = new Predicate() { + @Override + public boolean apply(@Nullable TestMethod input) { + return input != null; + } + }; public TestClassLoader(TestClassScanner scanner, TestClassFilter filter) { this.scanner = scanner; this.filter = filter; } - public List loadTestClasses() throws TestClassScanningException { + public List loadTestClasses() throws TestClassScanningException { List allTestClasses = scanner.scanForTestClasses(); - return filter.anyUserFilter(allTestClasses); + List testClasses = filter.anyUserFilter(allTestClasses); + return from(testClasses).transformAndConcat(new Function>() { + @Nullable + @Override + public Iterable apply(final @Nullable TestClass testClass) { + return (testClass != null && testClass.getMethods() != null) ? + from(testClass.getMethods()) + .filter(VALID_METHOD) + .transform(new Function() { + @Nullable + @Override + public TestCaseEvent apply(@Nullable TestMethod testMethod) { + assert testMethod != null; + return new TestCaseEvent(testMethod.getName(), + testClass.getName(), + testMethod.isIgnored()); + } + }) : Collections.emptyList(); + + } + }).toList(); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java index 9d1842fd..b701a0ab 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java @@ -35,6 +35,7 @@ public static HtmlSummary toHtmlSummary(Summary summary) { htmlSummary.subtitle = summary.getSubtitle(); htmlSummary.pools = transform(summary.getPoolSummaries(), toHtmlPoolSummary()); htmlSummary.ignoredTests = summary.getIgnoredTests(); + htmlSummary.failedTests = summary.getFailedTests(); htmlSummary.overallStatus = new OutcomeAggregator().aggregate(summary) ? "pass" : "fail"; return htmlSummary; } @@ -67,7 +68,7 @@ private static Function toHtmlTestResult(final Strin @Nullable public HtmlTestResult apply(@Nullable TestResult input) { HtmlTestResult htmlTestResult = new HtmlTestResult(); - htmlTestResult.status = input.getResultStatus().name().toLowerCase(); + htmlTestResult.status = computeStatus(input); htmlTestResult.prettyClassName = readableClassName(input.getTestClass()); htmlTestResult.prettyMethodName = readableTestMethodName(input.getTestMethod()); htmlTestResult.timeTaken = String.format("%.2f", input.getTimeTaken()); @@ -89,6 +90,15 @@ public HtmlTestResult apply(@Nullable TestResult input) { }; } + private static String computeStatus(@Nullable TestResult input) { + String result = input.getResultStatus().name().toLowerCase(); + if(input.getResultStatus() == ResultStatus.PASS + && input.getTotalFailureCount() > 0){ + result = "warn"; + } + return result; + } + public static Function toHtmlLogCatMessages() { return new Function() { @Nullable diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java index bc6fff30..9dca15eb 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java @@ -24,4 +24,5 @@ public class HtmlSummary { public String subtitle; public ArrayList ignoredTests; public String overallStatus; + public ArrayList failedTests; } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummaryPrinter.java b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummaryPrinter.java index 3ac07453..2405d30e 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummaryPrinter.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummaryPrinter.java @@ -41,6 +41,7 @@ public class HtmlSummaryPrinter implements SummaryPrinter { "ceiling_android.png", "ceiling_android-green.png", "ceiling_android-red.png", + "ceiling_android-yellow.png", "device.png", "icon-devices.png", "icon-log.png", diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java b/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java index fb8271d5..9eb64d82 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java @@ -43,7 +43,7 @@ public static Function toPoolOutcome() { public Boolean apply(@Nullable PoolSummary input) { final Collection testResults = input.getTestResults(); final Collection testOutcomes = transform(testResults, toTestOutcome()); - return and(testOutcomes); + return testOutcomes.size() > 0 && and(testOutcomes); } }; } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java b/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java index f4068a79..e66c798e 100644 --- a/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java @@ -11,7 +11,7 @@ package com.shazam.fork.summary; import com.shazam.fork.model.Pool; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import java.util.Collection; import java.util.List; @@ -28,8 +28,8 @@ public Summarizer(SummaryCompiler summaryCompiler, SummaryPrinter summaryPrinter this.outcomeAggregator = outcomeAggregator; } - boolean summarize(Collection pools, List testClasses) { - Summary summary = summaryCompiler.compileSummary(pools, testClasses); + boolean summarize(Collection pools, List testCases) { + Summary summary = summaryCompiler.compileSummary(pools, testCases); summaryPrinter.print(summary); return outcomeAggregator.aggregate(summary); } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java index 0a5b2c7d..d3d4dd38 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java @@ -12,9 +12,11 @@ */ package com.shazam.fork.summary; -import com.google.common.base.Function; +import com.google.common.collect.Lists; import com.shazam.fork.RuntimeConfiguration; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.runner.PoolTestRunner; import com.shazam.fork.system.io.FileManager; @@ -22,11 +24,9 @@ import org.simpleframework.xml.core.Persister; import java.io.File; -import java.util.*; +import java.util.Collection; +import java.util.List; -import javax.annotation.Nonnull; - -import static com.google.common.collect.Collections2.transform; import static com.shazam.fork.model.Device.Builder.aDevice; import static com.shazam.fork.summary.PoolSummary.Builder.aPoolSummary; import static com.shazam.fork.summary.Summary.Builder.aSummary; @@ -43,40 +43,41 @@ public class SummaryCompiler { public SummaryCompiler(RuntimeConfiguration runtimeConfiguration, FileManager fileManager) { this.runtimeConfiguration = runtimeConfiguration; this.fileManager = fileManager; - serializer = new Persister(); - } + serializer = new Persister(); + } - Summary compileSummary(Collection pools, List testClasses) { - Summary.Builder summaryBuilder = aSummary(); - for (Pool pool : pools) { - PoolSummary poolSummary = compilePoolSummary(pool); + Summary compileSummary(Collection pools, List testCases) { + Summary.Builder summaryBuilder = aSummary(); + for (Pool pool : pools) { + PoolSummary poolSummary = compilePoolSummary(pool, summaryBuilder); summaryBuilder.addPoolSummary(poolSummary); - } - addIgnoredTests(testClasses, summaryBuilder); + } + addIgnoredTests(testCases, summaryBuilder); summaryBuilder.withTitle(runtimeConfiguration.getTitle()); summaryBuilder.withSubtitle(runtimeConfiguration.getSubtitle()); - return summaryBuilder.build(); - } + return summaryBuilder.build(); + } - private PoolSummary compilePoolSummary(Pool pool) { + private PoolSummary compilePoolSummary(Pool pool, Summary.Builder summaryBuilder) { PoolSummary.Builder poolSummaryBuilder = aPoolSummary().withPoolName(pool.getName()); - for (Device device: pool.getDevices()) { - compileResultsForDevice(pool, poolSummaryBuilder, device); + for (Device device : pool.getDevices()) { + compileResultsForDevice(pool, poolSummaryBuilder, summaryBuilder, device); } Device watchdog = getPoolWatchdog(pool.getName()); - compileResultsForDevice(pool, poolSummaryBuilder, watchdog); + compileResultsForDevice(pool, poolSummaryBuilder, summaryBuilder, watchdog); return poolSummaryBuilder.build(); } - private void compileResultsForDevice(Pool pool, PoolSummary.Builder poolSummaryBuilder, Device device) { + private void compileResultsForDevice(Pool pool, PoolSummary.Builder poolSummaryBuilder, Summary.Builder summaryBuilder, Device device) { File[] deviceResultFiles = fileManager.getTestFilesForDevice(pool, device); if (deviceResultFiles == null) { return; } for (File file : deviceResultFiles) { - Collection testResults = parseTestResultsFromFile(file, device); - poolSummaryBuilder.addTestResults(testResults); + Collection testResult = parseTestResultsFromFile(file, device); + poolSummaryBuilder.addTestResults(testResult); + addFailedTests(testResult, summaryBuilder); } } @@ -88,41 +89,55 @@ private Device getPoolWatchdog(String poolName) { .build(); } - private void addIgnoredTests(List testClasses, Summary.Builder summaryBuilder) { - for (TestClass testClass : testClasses) { - for (TestMethod ignoredMethod : testClass.getIgnoredMethods()) { - summaryBuilder.addIgnoredTest(testClass.getName() + ":" + ignoredMethod); + private void addIgnoredTests(List testCases, Summary.Builder summaryBuilder) { + for (TestCaseEvent testCase : testCases) { + if (testCase.isIgnored()) { + summaryBuilder.addIgnoredTest(testCase.getTestClass() + ":" + testCase.getTestMethod()); + } + } + } + + private void addFailedTests(Collection testResults, Summary.Builder summaryBuilder) { + for (TestResult testResult : testResults) { + int totalFailureCount = testResult.getTotalFailureCount(); + if (totalFailureCount > 0) { + String failedTest = totalFailureCount + " times " + testResult.getTestClass() + + "#" + testResult.getTestMethod() + " on " + testResult.getDevice().getSerial() ; + summaryBuilder.addFailedTests(failedTest); } } } - private Collection parseTestResultsFromFile(File file, Device device) { - try { - TestSuite testSuite = serializer.read(TestSuite.class, file, STRICT); - List testCases = testSuite.getTestCases(); - if ((testCases == null) || testCases.isEmpty()) { - return new ArrayList<>(0); - } - return transform(testCases, toTestResult(device)); - } catch (Exception e) { - throw new RuntimeException("Error when parsing file: " + file.getAbsolutePath(), e); - } - } - - private Function toTestResult(final Device device) { - return new Function() { - @SuppressWarnings("NullableProblems") - @Override - public TestResult apply(@Nonnull TestCase testCase) { - return aTestResult() - .withDevice(device) - .withTestClass(testCase.getClassname()) - .withTestMethod(testCase.getName()) - .withTimeTaken(testCase.getTime()) - .withErrorTrace(testCase.getError()) - .withFailureTrace(testCase.getFailure()) - .build(); - } - }; - } + private Collection parseTestResultsFromFile(File file, Device device) { + try { + TestSuite testSuite = serializer.read(TestSuite.class, file, STRICT); + Collection testCases = testSuite.getTestCase(); + List result = Lists.newArrayList(); + if ((testCases == null)) { + return result; + } + + for(TestCase testCase : testCases){ + TestResult testResult = getTestResult(device, testSuite, testCase); + result.add(testResult); + } + return result; + } catch (Exception e) { + throw new RuntimeException("Error when parsing file: " + file.getAbsolutePath(), e); + } + } + + private TestResult getTestResult(Device device, TestSuite testSuite, TestCase testCase) { + TestResult.Builder testResultBuilder = aTestResult() + .withDevice(device) + .withTestClass(testCase.getClassname()) + .withTestMethod(testCase.getName()) + .withTimeTaken(testCase.getTime()) + .withErrorTrace(testCase.getError()) + .withFailureTrace(testCase.getFailure()); + if (testSuite.getProperties() != null) { + testResultBuilder.withTestMetrics(testSuite.getProperties()); + } + return testResultBuilder.build(); + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java index 573c1cd0..2bd0accb 100644 --- a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java @@ -13,7 +13,7 @@ package com.shazam.fork.summary; import com.shazam.fork.model.Pool; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +33,7 @@ public class SummaryGeneratorHook extends Thread { private final Summarizer summarizer; private Collection pools; - private List testClasses; + private List testCases; public SummaryGeneratorHook(Summarizer summarizer) { this.summarizer = summarizer; @@ -44,11 +44,11 @@ public SummaryGeneratorHook(Summarizer summarizer) { * shutdown hook. * * @param pools the pools to consider for the summary - * @param testClasses the test classes for the summary + * @param testCases the test cases for the summary */ - public void registerHook(Collection pools, List testClasses) { + public void registerHook(Collection pools, List testCases) { this.pools = pools; - this.testClasses = testClasses; + this.testCases = testCases; Runtime.getRuntime().addShutdownHook(this); } @@ -60,7 +60,7 @@ public void registerHook(Collection pools, List testClasses) { */ public boolean defineOutcome() { if (hasNotRunYet.compareAndSet(true, false)) { - return summarizer.summarize(pools, testClasses); + return summarizer.summarize(pools, testCases); } return false; } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java b/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java index 915ab5be..ecd22bc1 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java @@ -13,9 +13,12 @@ package com.shazam.fork.summary; import org.simpleframework.xml.ElementList; +import org.simpleframework.xml.ElementMap; +import org.simpleframework.xml.Path; import org.simpleframework.xml.Root; import java.util.List; +import java.util.Map; @Root @@ -24,7 +27,15 @@ class TestSuite { @ElementList(inline=true, type=TestCase.class, required=false) private List testCases; - public List getTestCases() { - return testCases; - } + @Path("./properties") + @ElementMap(required = false, entry = "property", key = "name", value = "value", attribute = true, inline = true) + private Map properties; + + public List getTestCase() { + return testCases; + } + + public Map getProperties() { + return properties; + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java b/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java index f930bcf3..c3c12a13 100644 --- a/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java +++ b/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java @@ -29,21 +29,15 @@ public FileManager(File output) { this.output = output; } - public File createFileForTest(Pool pool, Device device, TestClass testClass) { - try { - Path directory = createDirectory(TEST, pool, device); - String filename = createFilenameForTestClass(testClass, TEST); - return createFile(directory, filename); - } catch (IOException e) { - throw new CouldNotCreateDirectoryException(e); - } - } - public File[] getTestFilesForDevice(Pool pool, Device serial) { Path path = getDirectory(TEST, pool, serial); return path.toFile().listFiles(); } + public File createFile(FileType fileType, Pool pool, Device device, TestCaseEvent testCaseEvent){ + return createFile(fileType, pool, device, new TestIdentifier(testCaseEvent.getTestClass(), testCaseEvent.getTestMethod())); + } + public File createFile(FileType fileType, Pool pool, Device device, TestIdentifier testIdentifier, int sequenceNumber) { try { Path directory = createDirectory(fileType, pool, device); @@ -101,10 +95,6 @@ private File createFile(Path directory, String filename) { return new File(directory.toFile(), filename); } - private String createFilenameForTestClass(TestClass testClass, FileType fileType) { - return testClass.getName() + "." + fileType.getSuffix(); - } - private String createFilenameForTest(TestIdentifier testIdentifier, FileType fileType) { return String.format("%s.%s", testIdentifier.toString(), fileType.getSuffix()); } diff --git a/fork-runner/src/main/resources/forkpages/index.html b/fork-runner/src/main/resources/forkpages/index.html index 23086f78..5d1c7b5e 100755 --- a/fork-runner/src/main/resources/forkpages/index.html +++ b/fork-runner/src/main/resources/forkpages/index.html @@ -52,6 +52,16 @@

{{title}}

{{/ignoredTests}} + Failed tests: +
    + {{#failedTests}} +
  • {{.}}
    • + {{/failedTests}} + {{^failedTests}} +
  • None.
    • + {{/failedTests}} +
+