From 6d4960d091c3e09e92af519ac3a165755c0f4f7c Mon Sep 17 00:00:00 2001 From: Francesco Pezzato Date: Tue, 8 Mar 2016 11:31:14 +0000 Subject: [PATCH 01/22] Execute single test methods (instead of whole test classes) in RemoteAndroidTestRunner. --- .../main/java/com/shazam/fork/ForkRunner.java | 81 +++++++++++++++++-- .../fork/injector/ForkRunnerInjector.java | 8 +- .../runner/FailureAccumulatorInjector.java | 20 +++++ .../com/shazam/fork/model/TestCaseEvent.java | 26 ++++++ .../shazam/fork/runner/DeviceTestRunner.java | 14 ++-- .../fork/runner/DeviceTestRunnerFactory.java | 9 ++- .../fork/runner/FailureAccumulator.java | 41 ++++++++++ .../fork/runner/FailureAccumulatorImpl.java | 51 ++++++++++++ .../shazam/fork/runner/PoolTestRunner.java | 19 +++-- .../fork/runner/PoolTestRunnerFactory.java | 18 +++-- .../java/com/shazam/fork/runner/TestRun.java | 17 ++-- .../shazam/fork/runner/TestRunFactory.java | 6 +- .../shazam/fork/runner/TestRunParameters.java | 10 +-- .../listeners/ForkXmlTestRunListener.java | 9 ++- .../fork/runner/listeners/RetryListener.java | 33 ++++++++ .../listeners/TestRunListenersFactory.java | 21 +++-- .../shazam/fork/suite/TestClassLoader.java | 39 ++++++++- .../com/shazam/fork/summary/Summarizer.java | 6 +- .../shazam/fork/summary/SummaryCompiler.java | 12 +-- .../fork/summary/SummaryGeneratorHook.java | 12 +-- .../shazam/fork/system/io/FileManager.java | 18 +---- 21 files changed, 381 insertions(+), 89 deletions(-) create mode 100644 fork-runner/src/main/java/com/shazam/fork/injector/runner/FailureAccumulatorInjector.java create mode 100644 fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java create mode 100644 fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulator.java create mode 100644 fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulatorImpl.java create mode 100644 fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java diff --git a/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java b/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java index 7a8be308..03d5a63d 100755 --- a/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/ForkRunner.java @@ -12,11 +12,23 @@ */ package com.shazam.fork; +import com.android.ddmlib.testrunner.TestIdentifier; +import com.beust.jcommander.internal.Lists; +import com.google.common.base.Optional; +import com.google.common.base.Predicate; +import com.shazam.fork.injector.ConfigurationInjector; +import com.shazam.fork.model.Device; import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestMethod; import com.shazam.fork.pooling.NoDevicesForPoolException; import com.shazam.fork.pooling.PoolLoader; -import com.shazam.fork.runner.*; +import com.shazam.fork.runner.FailureAccumulator; +import com.shazam.fork.runner.FailureAccumulatorImpl; +import com.shazam.fork.runner.PoolTestRunner; +import com.shazam.fork.runner.PoolTestRunnerFactory; +import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.suite.TestClassLoader; import com.shazam.fork.suite.TestClassScanningException; import com.shazam.fork.summary.SummaryGeneratorHook; @@ -29,7 +41,11 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; +import javax.annotation.Nullable; + +import static com.google.common.collect.FluentIterable.from; import static com.shazam.fork.Utils.namedExecutor; +import static com.shazam.fork.model.Pool.Builder.aDevicePool; public class ForkRunner { private static final Logger logger = LoggerFactory.getLogger(ForkRunner.class); @@ -39,17 +55,37 @@ public class ForkRunner { private final PoolTestRunnerFactory poolTestRunnerFactory; private final ProgressReporter progressReporter; private final SummaryGeneratorHook summaryGeneratorHook; + private final FailureAccumulatorImpl failureAccumulator; public ForkRunner(PoolLoader poolLoader, TestClassLoader testClassLoader, PoolTestRunnerFactory poolTestRunnerFactory, ProgressReporter progressReporter, - SummaryGeneratorHook summaryGeneratorHook) { + SummaryGeneratorHook summaryGeneratorHook, FailureAccumulatorImpl failureAccumulator) { this.poolLoader = poolLoader; this.testClassLoader = testClassLoader; this.poolTestRunnerFactory = poolTestRunnerFactory; this.progressReporter = progressReporter; this.summaryGeneratorHook = summaryGeneratorHook; + this.failureAccumulator = failureAccumulator; + } + + private List convertToTestClass(Collection testIdentifiers) { + + List result = Lists.newArrayList(); + + for (TestIdentifier testIdentifier : testIdentifiers) { + TestMethod testMethod = TestMethod.Builder.testMethod() + .withName(testIdentifier.getTestName()).build(); + + TestClass testClass = TestClass.Builder.testClass() + .withName(testIdentifier.getClassName()) + .withMethods(Lists.newArrayList(testMethod)) + .build(); + result.add(testClass); + + } + return result; } public boolean run() { @@ -60,16 +96,23 @@ public boolean run() { CountDownLatch poolCountDownLatch = new CountDownLatch(numberOfPools); poolExecutor = namedExecutor(numberOfPools, "PoolExecutor-%d"); - List testClasses = testClassLoader.loadTestClasses(); - summaryGeneratorHook.registerHook(pools, testClasses); + List testCases = testClassLoader.loadTestClasses(); + summaryGeneratorHook.registerHook(pools, testCases); progressReporter.start(); for (Pool pool : pools) { - PoolTestRunner poolTestRunner = poolTestRunnerFactory.createPoolTestRunner(pool, testClasses, + PoolTestRunner poolTestRunner = poolTestRunnerFactory.createPoolTestRunner(pool, testCases, poolCountDownLatch, progressReporter); poolExecutor.execute(poolTestRunner); } poolCountDownLatch.await(); + + Configuration configuration = ConfigurationInjector.configuration(); + if(!failureAccumulator.isEmpty()){ + logger.info("Re-run failing tests."); + rerunFailures(poolExecutor, pools); + } + progressReporter.stop(); boolean overallSuccess = summaryGeneratorHook.defineOutcome(); @@ -90,4 +133,32 @@ public boolean run() { } } } + + private void rerunFailures(ExecutorService poolExecutor, Collection pools) throws InterruptedException { + for(final Device device : failureAccumulator.getFailures().keySet()){ + Collection testIdentifiers = failureAccumulator.getFailures().get(device); + + Optional poolWithDevice = from(pools).firstMatch(new Predicate() { + @Override + public boolean apply(@Nullable Pool input) { + return input != null && input.getDevices().contains(device); + } + }); + CountDownLatch retryCountDownLatch = new CountDownLatch(1); + if(poolWithDevice.isPresent()){ + Pool newPoolWithSingleDevice = aDevicePool().addDevice(device) + .withName(poolWithDevice.get().getName()) + .build(); + + PoolTestRunner poolTestRunner = poolTestRunnerFactory.createPoolTestRunner(newPoolWithSingleDevice, + convertToTestClass(testIdentifiers), + retryCountDownLatch, + progressReporter, + new FailureAccumulator.NoOpFailureAccumulator() + ); + poolExecutor.execute(poolTestRunner); + retryCountDownLatch.await(); + } + } + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java index 7467a9ee..f33e49c3 100644 --- a/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java +++ b/fork-runner/src/main/java/com/shazam/fork/injector/ForkRunnerInjector.java @@ -17,12 +17,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static com.shazam.fork.utils.Utils.millisSinceNanoTime; -import static com.shazam.fork.injector.runner.ProgressReporterInjector.progressReporter; import static com.shazam.fork.injector.pooling.PoolLoaderInjector.poolLoader; +import static com.shazam.fork.injector.runner.FailureAccumulatorInjector.failureAccumulator; import static com.shazam.fork.injector.runner.PoolTestRunnerFactoryInjector.poolTestRunnerFactory; +import static com.shazam.fork.injector.runner.ProgressReporterInjector.progressReporter; import static com.shazam.fork.injector.suite.TestClassLoaderInjector.testClassLoader; import static com.shazam.fork.injector.summary.SummaryGeneratorHookInjector.summaryGeneratorHook; +import static com.shazam.fork.utils.Utils.millisSinceNanoTime; import static java.lang.System.nanoTime; public class ForkRunnerInjector { @@ -37,7 +38,8 @@ public static ForkRunner forkRunner() { testClassLoader(), poolTestRunnerFactory(), progressReporter(), - summaryGeneratorHook()); + summaryGeneratorHook(), + failureAccumulator()); logger.debug("Bootstrap of ForkRunner took: {} milliseconds", millisSinceNanoTime(startNanos)); diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/runner/FailureAccumulatorInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/runner/FailureAccumulatorInjector.java new file mode 100644 index 00000000..919e9a68 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/injector/runner/FailureAccumulatorInjector.java @@ -0,0 +1,20 @@ +/* + * Copyright 2016 Shazam Entertainment Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.shazam.fork.injector.runner; + +import com.shazam.fork.runner.FailureAccumulatorImpl; + +public class FailureAccumulatorInjector { + + public static FailureAccumulatorImpl failureAccumulator() { + return new FailureAccumulatorImpl(); + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java new file mode 100644 index 00000000..483170aa --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java @@ -0,0 +1,26 @@ +package com.shazam.fork.model; + +public class TestCaseEvent { + + private final String testMethod; + private final String testClass; + private final boolean isIgnored; + + public TestCaseEvent(String testMethod, String testClass, boolean isIgnored) { + this.testMethod = testMethod; + this.testClass = testClass; + this.isIgnored = isIgnored; + } + + public String getTestMethod() { + return testMethod; + } + + public String getTestClass() { + return testClass; + } + + public boolean isIgnored() { + return isIgnored; + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java index 804ea492..ff95d6f2 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java @@ -31,18 +31,19 @@ public class DeviceTestRunner implements Runnable { private final Installer installer; private final Pool pool; private final Device device; - private final Queue queueOfTestsInPool; + private final Queue queueOfTestsInPool; private final CountDownLatch deviceCountDownLatch; private final ProgressReporter progressReporter; private final TestRunFactory testRunFactory; + private final FailureAccumulator failureAccumulator; public DeviceTestRunner(Installer installer, Pool pool, Device device, - Queue queueOfTestsInPool, + Queue queueOfTestsInPool, CountDownLatch deviceCountDownLatch, ProgressReporter progressReporter, - TestRunFactory testRunFactory) { + TestRunFactory testRunFactory, FailureAccumulator failureAccumulator) { this.installer = installer; this.pool = pool; this.device = device; @@ -50,6 +51,7 @@ public DeviceTestRunner(Installer installer, this.deviceCountDownLatch = deviceCountDownLatch; this.progressReporter = progressReporter; this.testRunFactory = testRunFactory; + this.failureAccumulator = failureAccumulator; } @Override @@ -61,9 +63,9 @@ public void run() { removeRemoteDirectory(deviceInterface); createRemoteDirectory(deviceInterface); - TestClass testClass; - while ((testClass = queueOfTestsInPool.poll()) != null) { - TestRun testRun = testRunFactory.createTestRun(testClass, device, pool, progressReporter); + TestCaseEvent testCaseEvent; + while ((testCaseEvent = queueOfTestsInPool.poll()) != null) { + TestRun testRun = testRunFactory.createTestRun(testCaseEvent, device, pool, progressReporter); testRun.execute(); } } finally { diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java index 1ad57676..bd2f5f3f 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunnerFactory.java @@ -27,10 +27,12 @@ public DeviceTestRunnerFactory(Installer installer, TestRunFactory testRunFactor } public Runnable createDeviceTestRunner(Pool pool, - Queue testClassQueue, + Queue testClassQueue, CountDownLatch deviceInPoolCountDownLatch, Device device, - ProgressReporter progressReporter) { + ProgressReporter progressReporter, + FailureAccumulator failureAccumulator + ) { return new DeviceTestRunner( installer, pool, @@ -38,6 +40,7 @@ public Runnable createDeviceTestRunner(Pool pool, testClassQueue, deviceInPoolCountDownLatch, progressReporter, - testRunFactory); + testRunFactory, + failureAccumulator); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulator.java b/fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulator.java new file mode 100644 index 00000000..5f728aee --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulator.java @@ -0,0 +1,41 @@ +/* + * Copyright 2016 Shazam Entertainment Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +package com.shazam.fork.runner; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.shazam.fork.model.Device; + +import java.util.Collection; +import java.util.Map; + +public interface FailureAccumulator { + void testFailed(Device device, TestIdentifier failedTest); + + boolean isEmpty(); + + Map> getFailures(); + + class NoOpFailureAccumulator implements FailureAccumulator { + + @Override + public void testFailed(Device device, TestIdentifier failedTest) { + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public Map> getFailures() { + return null; + } + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulatorImpl.java b/fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulatorImpl.java new file mode 100644 index 00000000..5da5a023 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/runner/FailureAccumulatorImpl.java @@ -0,0 +1,51 @@ +/* + * Copyright 2016 Shazam Entertainment Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ +package com.shazam.fork.runner; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.google.common.collect.Lists; +import com.shazam.fork.model.Device; + +import java.util.Collection; +import java.util.Map; + +import static com.google.common.collect.ImmutableMap.copyOf; +import static com.google.common.collect.Maps.newHashMap; + +public class FailureAccumulatorImpl implements FailureAccumulator { + + private final Map> failures; + + public FailureAccumulatorImpl() { + failures = newHashMap(); + } + + @Override + public void testFailed(Device device, TestIdentifier failedTest) { + ensureDevice(device); + failures.get(device).add(failedTest); + } + + private void ensureDevice(Device device) { + if (!failures.containsKey(device)) { + failures.put(device, Lists.newArrayList()); + } + } + + @Override + public boolean isEmpty(){ + return failures.isEmpty(); + } + + @Override + public Map> getFailures() { + return copyOf(failures); + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java index 5df6ed3d..d0521092 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java @@ -36,24 +36,26 @@ public class PoolTestRunner implements Runnable { private final Configuration configuration; private final FileManager fileManager; private final Pool pool; - private final Queue testClasses; + private final Queue testCases; private final CountDownLatch poolCountDownLatch; private final DeviceTestRunnerFactory deviceTestRunnerFactory; private final ProgressReporter progressReporter; + private final FailureAccumulator failureAccumulator; public PoolTestRunner(Configuration configuration, FileManager fileManager, DeviceTestRunnerFactory deviceTestRunnerFactory, Pool pool, - Queue testClasses, + Queue testCases, CountDownLatch poolCountDownLatch, - ProgressReporter progressReporter) { + ProgressReporter progressReporter, FailureAccumulator failureAccumulator) { this.configuration = configuration; this.fileManager = fileManager; this.pool = pool; - this.testClasses = testClasses; + this.testCases = testCases; this.poolCountDownLatch = poolCountDownLatch; this.deviceTestRunnerFactory = deviceTestRunnerFactory; this.progressReporter = progressReporter; + this.failureAccumulator = failureAccumulator; } public void run() { @@ -65,7 +67,7 @@ public void run() { CountDownLatch deviceCountDownLatch = new CountDownLatch(devicesInPool); logger.info("Pool {} started", poolName); for (Device device : pool.getDevices()) { - Runnable deviceTestRunner = deviceTestRunnerFactory.createDeviceTestRunner(pool, testClasses, + Runnable deviceTestRunner = deviceTestRunnerFactory.createDeviceTestRunner(pool, testCases, deviceCountDownLatch, device, progressReporter); concurrentDeviceExecutor.execute(deviceTestRunner); } @@ -73,7 +75,7 @@ public void run() { } catch (InterruptedException e) { logger.warn("Pool {} was interrupted while running", poolName); } finally { - failAnyDroppedClasses(pool, testClasses); + //failAnyDroppedClasses(pool, testCases); if (concurrentDeviceExecutor != null) { concurrentDeviceExecutor.shutdown(); } @@ -89,15 +91,16 @@ public void run() { *

* In particular, not triggering the console listener will probably make the flaky report better. */ + //TODO - verify if this is still useful. private void failAnyDroppedClasses(Pool pool, Queue testClassQueue) { - HashMap emptyHash = new HashMap<>(); + HashMap emptyHash = new HashMap(); TestClass nextTest; while ((nextTest = testClassQueue.poll()) != null) { String className = nextTest.getName(); String poolName = pool.getName(); Device failedTestsDevice = aDevice().withSerial(DROPPED_BY + poolName).build(); ForkXmlTestRunListener xmlGenerator = getForkXmlTestRunListener(fileManager, configuration.getOutput(), - pool, failedTestsDevice, nextTest); + pool, failedTestsDevice, null/*nextTest*/); //TODO Collection methods = nextTest.getUnignoredMethods(); xmlGenerator.testRunStarted(poolName, methods.size()); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java index fbfbe30d..d8f9c7ba 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java @@ -12,7 +12,7 @@ import com.shazam.fork.Configuration; import com.shazam.fork.model.Pool; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.system.io.FileManager; import java.util.LinkedList; @@ -33,11 +33,12 @@ public PoolTestRunnerFactory(Configuration configuration, } public PoolTestRunner createPoolTestRunner(Pool pool, - List testClasses, + List testCases, CountDownLatch poolCountDownLatch, - ProgressReporter progressReporter) { + ProgressReporter progressReporter, + FailureAccumulator failureAccumulator) { - int totalTests = countTests(testClasses); + int totalTests = testCases.size(); progressReporter.addPoolProgress(pool, new PoolProgressTracker(totalTests)); return new PoolTestRunner( @@ -45,16 +46,17 @@ public PoolTestRunner createPoolTestRunner(Pool pool, fileManager, deviceTestRunnerFactory, pool, - new LinkedList<>(testClasses), + new LinkedList<>(testCases), poolCountDownLatch, - progressReporter); + progressReporter, + failureAccumulator); } - private int countTests(List testClasses) { + /* private int countTests(List testClasses) { int sum = 0; for (TestClass testClass : testClasses) { sum += testClass.getMethods().size(); } return sum; - } + }*/ } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java index 6467dca5..f19e3c80 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java @@ -12,9 +12,13 @@ */ package com.shazam.fork.runner; -import com.android.ddmlib.*; -import com.android.ddmlib.testrunner.*; -import com.shazam.fork.model.TestClass; +import com.android.ddmlib.AdbCommandRejectedException; +import com.android.ddmlib.ShellCommandUnresponsiveException; +import com.android.ddmlib.TimeoutException; +import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; +import com.android.ddmlib.testrunner.ITestRunListener; +import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; +import com.shazam.fork.model.TestCaseEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,14 +48,15 @@ public void execute() { testRunParameters.getTestRunner(), testRunParameters.getDeviceInterface()); - TestClass test = testRunParameters.getTest(); - String testClassName = test.getName(); + TestCaseEvent test = testRunParameters.getTest(); + String testClassName = test.getTestClass(); + String testMethodName = test.getTestMethod(); IRemoteAndroidTestRunner.TestSize testSize = testRunParameters.getTestSize(); if (testSize != null) { runner.setTestSize(testSize); } runner.setRunName(poolName); - runner.setClassName(testClassName); + runner.setMethodName(testClassName, testMethodName); runner.setMaxtimeToOutputResponse(testRunParameters.getTestOutputTimeout()); try { runner.run(testRunListeners); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java index c90118de..c4339af9 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java @@ -34,12 +34,12 @@ public TestRunFactory(Configuration configuration, this.testRunListenersFactory = testRunListenersFactory; } - public TestRun createTestRun(TestClass testClass, Device device, Pool pool, ProgressReporter progressReporter) { + public TestRun createTestRun(TestCaseEvent testCase, Device device, Pool pool, ProgressReporter progressReporter) { InstrumentationInfo instrumentationInfo = configuration.getInstrumentationInfo(); TestRunParameters testRunParameters = testRunParameters() .withDeviceInterface(device.getDeviceInterface()) - .withTest(testClass) + .withTest(testCase) .withTestPackage(instrumentationInfo.getInstrumentationPackage()) .withTestRunner(instrumentationInfo.getTestRunnerClass()) .withTestSize(runtimeConfiguration.getTestSize()) @@ -47,7 +47,7 @@ public TestRun createTestRun(TestClass testClass, Device device, Pool pool, Prog .build(); List testRunListeners = testRunListenersFactory.createTestListeners( - testClass, + testCase, device, pool, progressReporter); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java index dcd599a0..6a236d13 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunParameters.java @@ -14,19 +14,19 @@ import com.android.ddmlib.IDevice; import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import javax.annotation.Nullable; public class TestRunParameters { - private final TestClass test; + private final TestCaseEvent test; private final String testPackage; private final String testRunner; private final IRemoteAndroidTestRunner.TestSize testSize; private final int testOutputTimeout; private final IDevice deviceInterface; - public TestClass getTest() { + public TestCaseEvent getTest() { return test; } @@ -52,7 +52,7 @@ public IDevice getDeviceInterface() { } public static class Builder { - private TestClass test; + private TestCaseEvent test; private String testPackage; private String testRunner; private IRemoteAndroidTestRunner.TestSize testSize; @@ -63,7 +63,7 @@ public static Builder testRunParameters() { return new Builder(); } - public Builder withTest(TestClass test) { + public Builder withTest(TestCaseEvent test) { this.test = test; return this; } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java index 67a7f96d..288f894f 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java @@ -15,6 +15,7 @@ import com.android.ddmlib.testrunner.XmlTestRunListener; import com.shazam.fork.model.*; import com.shazam.fork.system.io.FileManager; +import com.shazam.fork.system.io.FileType; import java.io.File; @@ -22,17 +23,17 @@ public class ForkXmlTestRunListener extends XmlTestRunListener { private final FileManager fileManager; private final Pool pool; private final Device device; - private final TestClass testClass; + private final TestCaseEvent testCase; - public ForkXmlTestRunListener(FileManager fileManager, Pool pool, Device device, TestClass testClass) { + public ForkXmlTestRunListener(FileManager fileManager, Pool pool, Device device, TestCaseEvent testCase) { this.fileManager = fileManager; this.pool = pool; this.device = device; - this.testClass = testClass; + this.testCase = testCase; } @Override protected File getResultFile(File reportDir) { - return fileManager.createFileForTest(pool, device, testClass); + return fileManager.createFile(FileType.TEST, pool, device, testCase); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java new file mode 100644 index 00000000..9b6ca931 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java @@ -0,0 +1,33 @@ +/* + * Copyright 2016 Shazam Entertainment Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. + * + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + */ + +package com.shazam.fork.runner.listeners; + +import com.android.ddmlib.testrunner.TestIdentifier; +import com.shazam.fork.model.Device; +import com.shazam.fork.runner.FailureAccumulator; + +public class RetryListener extends NoOpITestRunListener { + + private final Device device; + private final FailureAccumulator failureAccumulator; + + public RetryListener(FailureAccumulator failureAccumulator, Device device) { + this.failureAccumulator = failureAccumulator; + this.device = device; + } + + @Override + public void testFailed(TestIdentifier test, String trace) { + failureAccumulator.testFailed(device, test); + } + +} + diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java index fea49059..2c81b45c 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java @@ -13,7 +13,10 @@ import com.android.ddmlib.testrunner.ITestRunListener; import com.google.gson.Gson; import com.shazam.fork.Configuration; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestClass; +import com.shazam.fork.runner.FailureAccumulator; import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.system.io.FileManager; @@ -38,26 +41,30 @@ public TestRunListenersFactory(Configuration configuration, this.gson = gson; } - public List createTestListeners(TestClass testClass, + public List createTestListeners(TestCaseEvent testCase, Device device, Pool pool, - ProgressReporter progressReporter) { + ProgressReporter progressReporter, + FailureAccumulator failureAccumulator + ) { return asList( new ProgressTestRunListener(pool, progressReporter), - getForkXmlTestRunListener(fileManager, configuration.getOutput(), pool, device, testClass), + getForkXmlTestRunListener(fileManager, configuration.getOutput(), pool, device, testCase), new ConsoleLoggingTestRunListener(configuration.getTestPackage(), device.getSerial(), device.getModelName(), progressReporter), new LogCatTestRunListener(gson, fileManager, pool, device), new SlowWarningTestRunListener(), - getScreenTraceTestRunListener(fileManager, pool, device)); + getScreenTraceTestRunListener(fileManager, pool, device), + new RetryListener(failureAccumulator, device) + ); } public static ForkXmlTestRunListener getForkXmlTestRunListener(FileManager fileManager, File output, Pool pool, Device device, - TestClass testClass) { - ForkXmlTestRunListener xmlTestRunListener = new ForkXmlTestRunListener(fileManager, pool, device, testClass); + TestCaseEvent testCase) { + ForkXmlTestRunListener xmlTestRunListener = new ForkXmlTestRunListener(fileManager, pool, device, testCase); xmlTestRunListener.setReportDir(output); return xmlTestRunListener; } diff --git a/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java b/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java index 1e07da88..e71ab06b 100644 --- a/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java +++ b/fork-runner/src/main/java/com/shazam/fork/suite/TestClassLoader.java @@ -10,21 +10,56 @@ package com.shazam.fork.suite; +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestMethod; +import java.util.Collections; import java.util.List; +import javax.annotation.Nullable; + +import static com.google.common.collect.FluentIterable.from; + public class TestClassLoader { private final TestClassScanner scanner; private final TestClassFilter filter; + private Predicate VALID_METHOD = new Predicate() { + @Override + public boolean apply(@Nullable TestMethod input) { + return input != null; + } + }; public TestClassLoader(TestClassScanner scanner, TestClassFilter filter) { this.scanner = scanner; this.filter = filter; } - public List loadTestClasses() throws TestClassScanningException { + public List loadTestClasses() throws TestClassScanningException { List allTestClasses = scanner.scanForTestClasses(); - return filter.anyUserFilter(allTestClasses); + List testClasses = filter.anyUserFilter(allTestClasses); + return from(testClasses).transformAndConcat(new Function>() { + @Nullable + @Override + public Iterable apply(final @Nullable TestClass testClass) { + return (testClass != null && testClass.getMethods() != null) ? + from(testClass.getMethods()) + .filter(VALID_METHOD) + .transform(new Function() { + @Nullable + @Override + public TestCaseEvent apply(@Nullable TestMethod testMethod) { + assert testMethod != null; + return new TestCaseEvent(testMethod.getName(), + testClass.getName(), + testMethod.isIgnored()); + } + }) : Collections.emptyList(); + + } + }).toList(); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java b/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java index f4068a79..e66c798e 100644 --- a/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/Summarizer.java @@ -11,7 +11,7 @@ package com.shazam.fork.summary; import com.shazam.fork.model.Pool; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import java.util.Collection; import java.util.List; @@ -28,8 +28,8 @@ public Summarizer(SummaryCompiler summaryCompiler, SummaryPrinter summaryPrinter this.outcomeAggregator = outcomeAggregator; } - boolean summarize(Collection pools, List testClasses) { - Summary summary = summaryCompiler.compileSummary(pools, testClasses); + boolean summarize(Collection pools, List testCases) { + Summary summary = summaryCompiler.compileSummary(pools, testCases); summaryPrinter.print(summary); return outcomeAggregator.aggregate(summary); } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java index 0a5b2c7d..381c45d6 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java @@ -46,13 +46,13 @@ public SummaryCompiler(RuntimeConfiguration runtimeConfiguration, FileManager fi serializer = new Persister(); } - Summary compileSummary(Collection pools, List testClasses) { + Summary compileSummary(Collection pools, List testCases) { Summary.Builder summaryBuilder = aSummary(); for (Pool pool : pools) { PoolSummary poolSummary = compilePoolSummary(pool); summaryBuilder.addPoolSummary(poolSummary); } - addIgnoredTests(testClasses, summaryBuilder); + addIgnoredTests(testCases, summaryBuilder); summaryBuilder.withTitle(runtimeConfiguration.getTitle()); summaryBuilder.withSubtitle(runtimeConfiguration.getSubtitle()); @@ -88,10 +88,10 @@ private Device getPoolWatchdog(String poolName) { .build(); } - private void addIgnoredTests(List testClasses, Summary.Builder summaryBuilder) { - for (TestClass testClass : testClasses) { - for (TestMethod ignoredMethod : testClass.getIgnoredMethods()) { - summaryBuilder.addIgnoredTest(testClass.getName() + ":" + ignoredMethod); + private void addIgnoredTests(List testCases, Summary.Builder summaryBuilder) { + for (TestCaseEvent testCase : testCases) { + if (testCase.isIgnored()) { + summaryBuilder.addIgnoredTest(testCase.getTestClass() + ":" + testCase.getTestMethod()); } } } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java index 573c1cd0..2bd0accb 100644 --- a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryGeneratorHook.java @@ -13,7 +13,7 @@ package com.shazam.fork.summary; import com.shazam.fork.model.Pool; -import com.shazam.fork.model.TestClass; +import com.shazam.fork.model.TestCaseEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +33,7 @@ public class SummaryGeneratorHook extends Thread { private final Summarizer summarizer; private Collection pools; - private List testClasses; + private List testCases; public SummaryGeneratorHook(Summarizer summarizer) { this.summarizer = summarizer; @@ -44,11 +44,11 @@ public SummaryGeneratorHook(Summarizer summarizer) { * shutdown hook. * * @param pools the pools to consider for the summary - * @param testClasses the test classes for the summary + * @param testCases the test cases for the summary */ - public void registerHook(Collection pools, List testClasses) { + public void registerHook(Collection pools, List testCases) { this.pools = pools; - this.testClasses = testClasses; + this.testCases = testCases; Runtime.getRuntime().addShutdownHook(this); } @@ -60,7 +60,7 @@ public void registerHook(Collection pools, List testClasses) { */ public boolean defineOutcome() { if (hasNotRunYet.compareAndSet(true, false)) { - return summarizer.summarize(pools, testClasses); + return summarizer.summarize(pools, testCases); } return false; } diff --git a/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java b/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java index f930bcf3..c3c12a13 100644 --- a/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java +++ b/fork-runner/src/main/java/com/shazam/fork/system/io/FileManager.java @@ -29,21 +29,15 @@ public FileManager(File output) { this.output = output; } - public File createFileForTest(Pool pool, Device device, TestClass testClass) { - try { - Path directory = createDirectory(TEST, pool, device); - String filename = createFilenameForTestClass(testClass, TEST); - return createFile(directory, filename); - } catch (IOException e) { - throw new CouldNotCreateDirectoryException(e); - } - } - public File[] getTestFilesForDevice(Pool pool, Device serial) { Path path = getDirectory(TEST, pool, serial); return path.toFile().listFiles(); } + public File createFile(FileType fileType, Pool pool, Device device, TestCaseEvent testCaseEvent){ + return createFile(fileType, pool, device, new TestIdentifier(testCaseEvent.getTestClass(), testCaseEvent.getTestMethod())); + } + public File createFile(FileType fileType, Pool pool, Device device, TestIdentifier testIdentifier, int sequenceNumber) { try { Path directory = createDirectory(fileType, pool, device); @@ -101,10 +95,6 @@ private File createFile(Path directory, String filename) { return new File(directory.toFile(), filename); } - private String createFilenameForTestClass(TestClass testClass, FileType fileType) { - return testClass.getName() + "." + fileType.getSuffix(); - } - private String createFilenameForTest(TestIdentifier testIdentifier, FileType fileType) { return String.format("%s.%s", testIdentifier.toString(), fileType.getSuffix()); } From 161b2b6ed000ddaeaf814b1be180eeb9401e1446 Mon Sep 17 00:00:00 2001 From: Francesco Pezzato Date: Tue, 8 Mar 2016 11:33:00 +0000 Subject: [PATCH 02/22] Enqueue again failed tests cases. --- .../shazam/fork/runner/DeviceTestRunner.java | 27 +++++++---- .../fork/runner/OverallProgressReporter.java | 6 +++ .../shazam/fork/runner/ProgressReporter.java | 2 + .../com/shazam/fork/runner/RetryWatchdog.java | 12 +++++ .../shazam/fork/runner/TestRunFactory.java | 17 +++++-- .../listeners/ForkXmlTestRunListener.java | 29 +++++++++++- .../fork/runner/listeners/RetryListener.java | 46 ++++++++++++++++--- .../listeners/TestRunListenersFactory.java | 8 ++-- 8 files changed, 122 insertions(+), 25 deletions(-) create mode 100644 fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java index ff95d6f2..6f8af259 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/DeviceTestRunner.java @@ -13,7 +13,9 @@ package com.shazam.fork.runner; import com.android.ddmlib.IDevice; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.system.adb.Installer; import org.slf4j.Logger; @@ -45,8 +47,8 @@ public DeviceTestRunner(Installer installer, ProgressReporter progressReporter, TestRunFactory testRunFactory, FailureAccumulator failureAccumulator) { this.installer = installer; - this.pool = pool; - this.device = device; + this.pool = pool; + this.device = device; this.queueOfTestsInPool = queueOfTestsInPool; this.deviceCountDownLatch = deviceCountDownLatch; this.progressReporter = progressReporter; @@ -54,8 +56,8 @@ public DeviceTestRunner(Installer installer, this.failureAccumulator = failureAccumulator; } - @Override - public void run() { + @Override + public void run() { IDevice deviceInterface = device.getDeviceInterface(); try { installer.prepareInstallation(deviceInterface); @@ -65,12 +67,17 @@ public void run() { TestCaseEvent testCaseEvent; while ((testCaseEvent = queueOfTestsInPool.poll()) != null) { - TestRun testRun = testRunFactory.createTestRun(testCaseEvent, device, pool, progressReporter); + TestRun testRun = testRunFactory.createTestRun(testCaseEvent, + device, + pool, + progressReporter, + queueOfTestsInPool, + testCaseEvent); testRun.execute(); } - } finally { + } finally { logger.info("Device {} from pool {} finished", device.getSerial(), pool.getName()); - deviceCountDownLatch.countDown(); - } - } + deviceCountDownLatch.countDown(); + } + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java index 318273cf..b7ccdbd7 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java @@ -26,6 +26,7 @@ public class OverallProgressReporter implements ProgressReporter { private long startOfTests; private long endOfTests; private final Map poolProgressTrackers = new HashMap<>(); + private final RetryWatchdog retryWatchdog = new RetryWatchdog(); @Override public void start() { @@ -76,4 +77,9 @@ public float getProgress() { return progress / size; } + @Override + public RetryWatchdog retryWatchdog() { + return retryWatchdog; + } + } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java index a2200ced..bc197139 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java @@ -32,4 +32,6 @@ public interface ProgressReporter { int getFailures(); float getProgress(); + + RetryWatchdog retryWatchdog(); } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java b/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java new file mode 100644 index 00000000..e84d9c28 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java @@ -0,0 +1,12 @@ +package com.shazam.fork.runner; + +import java.util.concurrent.atomic.AtomicInteger; + +public class RetryWatchdog { + + private AtomicInteger counter = new AtomicInteger(3); + + public boolean allowRetry() { + return counter.get() >= 0 && counter.getAndDecrement() > 0; + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java index c4339af9..5c8f6f45 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRunFactory.java @@ -13,10 +13,14 @@ import com.android.ddmlib.testrunner.ITestRunListener; import com.shazam.fork.Configuration; import com.shazam.fork.RuntimeConfiguration; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.InstrumentationInfo; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.runner.listeners.TestRunListenersFactory; import java.util.List; +import java.util.Queue; import static com.shazam.fork.runner.TestRunParameters.Builder.testRunParameters; @@ -34,7 +38,12 @@ public TestRunFactory(Configuration configuration, this.testRunListenersFactory = testRunListenersFactory; } - public TestRun createTestRun(TestCaseEvent testCase, Device device, Pool pool, ProgressReporter progressReporter) { + public TestRun createTestRun(TestCaseEvent testCase, + Device device, + Pool pool, + ProgressReporter progressReporter, + Queue queueOfTestsInPool, + TestCaseEvent currentTestCaseEvent) { InstrumentationInfo instrumentationInfo = configuration.getInstrumentationInfo(); TestRunParameters testRunParameters = testRunParameters() @@ -50,7 +59,9 @@ public TestRun createTestRun(TestCaseEvent testCase, Device device, Pool pool, P testCase, device, pool, - progressReporter); + progressReporter, + queueOfTestsInPool, + currentTestCaseEvent); return new TestRun( pool.getName(), diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java index 288f894f..055dc507 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java @@ -12,18 +12,29 @@ */ package com.shazam.fork.runner.listeners; +import com.android.ddmlib.testrunner.TestIdentifier; import com.android.ddmlib.testrunner.XmlTestRunListener; -import com.shazam.fork.model.*; +import com.google.common.collect.ImmutableMap; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.system.io.FileManager; import com.shazam.fork.system.io.FileType; import java.io.File; +import java.util.Map; + +import javax.annotation.Nonnull; + +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; public class ForkXmlTestRunListener extends XmlTestRunListener { private final FileManager fileManager; private final Pool pool; private final Device device; private final TestCaseEvent testCase; + private @Nonnull Boolean failedTest = FALSE; public ForkXmlTestRunListener(FileManager fileManager, Pool pool, Device device, TestCaseEvent testCase) { this.fileManager = fileManager; @@ -36,4 +47,20 @@ public ForkXmlTestRunListener(FileManager fileManager, Pool pool, Device device, protected File getResultFile(File reportDir) { return fileManager.createFile(FileType.TEST, pool, device, testCase); } + + @Override + public void testFailed(TestIdentifier test, String trace) { + this.failedTest = TRUE; + super.testFailed(test, trace); + } + + @Override + protected Map getPropertiesAttributes() { + Map superAttribs = super.getPropertiesAttributes(); + return ImmutableMap.builder() + .putAll(superAttribs) + .put("hasFailed", failedTest.toString()) + .build(); + + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java index 9b6ca931..ab442169 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java @@ -12,22 +12,54 @@ import com.android.ddmlib.testrunner.TestIdentifier; import com.shazam.fork.model.Device; -import com.shazam.fork.runner.FailureAccumulator; +import com.shazam.fork.model.TestCaseEvent; +import com.shazam.fork.runner.ProgressReporter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Queue; + +import javax.annotation.Nonnull; + +import static com.beust.jcommander.internal.Lists.newLinkedList; +import static com.google.common.base.Preconditions.checkNotNull; public class RetryListener extends NoOpITestRunListener { + private static final Logger logger = LoggerFactory.getLogger(RetryListener.class); + + @Nonnull private final Device device; - private final FailureAccumulator failureAccumulator; + @Nonnull + private final Queue failedTests = newLinkedList(); + @Nonnull + private final Queue queueOfTestsInPool; + @Nonnull + private final TestCaseEvent currentTestCaseEvent; + private ProgressReporter progressReporter; - public RetryListener(FailureAccumulator failureAccumulator, Device device) { - this.failureAccumulator = failureAccumulator; + public RetryListener(@Nonnull Device device, + @Nonnull Queue queueOfTestsInPool, + @Nonnull TestCaseEvent currentTestCaseEvent, + ProgressReporter progressReporter) { + checkNotNull(device); + checkNotNull(queueOfTestsInPool); + checkNotNull(currentTestCaseEvent); this.device = device; + this.queueOfTestsInPool = queueOfTestsInPool; + this.currentTestCaseEvent = currentTestCaseEvent; + this.progressReporter = progressReporter; } @Override public void testFailed(TestIdentifier test, String trace) { - failureAccumulator.testFailed(device, test); + failedTests.add(test); + if (progressReporter.retryWatchdog().allowRetry()) { + queueOfTestsInPool.add(currentTestCaseEvent); + logger.info("Test " + test.toString() + " enqueued again into device." + device.getSerial()); + } else { + logger.debug("Test " + test.toString() + " failed on device " + device.getSerial() + " but retry is not allowed."); + } } - } - diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java index 2c81b45c..8d1ce3ab 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java @@ -22,6 +22,7 @@ import java.io.File; import java.util.List; +import java.util.Queue; import static com.shazam.fork.model.Diagnostics.SCREENSHOTS; import static com.shazam.fork.model.Diagnostics.VIDEO; @@ -45,8 +46,8 @@ public List createTestListeners(TestCaseEvent testCase, Device device, Pool pool, ProgressReporter progressReporter, - FailureAccumulator failureAccumulator - ) { + Queue testCaseEventQueue, + TestCaseEvent currentTestCaseEvent) { return asList( new ProgressTestRunListener(pool, progressReporter), getForkXmlTestRunListener(fileManager, configuration.getOutput(), pool, device, testCase), @@ -55,8 +56,7 @@ public List createTestListeners(TestCaseEvent testCase, new LogCatTestRunListener(gson, fileManager, pool, device), new SlowWarningTestRunListener(), getScreenTraceTestRunListener(fileManager, pool, device), - new RetryListener(failureAccumulator, device) - ); + new RetryListener(device, testCaseEventQueue, currentTestCaseEvent, progressReporter)); } public static ForkXmlTestRunListener getForkXmlTestRunListener(FileManager fileManager, From ceb1eca0768b3dbe1216d115647fb48d66134542 Mon Sep 17 00:00:00 2001 From: Francesco Pezzato Date: Tue, 8 Mar 2016 11:33:19 +0000 Subject: [PATCH 03/22] Record in .xml output tests the failure count. --- .../java/com/shazam/fork/model/Counter.java | 47 ++++++++++++++ .../fork/model/DeviceTestCaseAccumulator.java | 64 +++++++++++++++++++ .../com/shazam/fork/model/TestCaseEvent.java | 18 ++++++ .../fork/model/TestCaseEventCounter.java | 11 ++++ .../fork/runner/OverallProgressReporter.java | 13 ++++ .../shazam/fork/runner/PoolTestRunner.java | 2 +- .../shazam/fork/runner/ProgressReporter.java | 6 ++ .../com/shazam/fork/runner/RetryWatchdog.java | 2 +- .../listeners/ForkXmlTestRunListener.java | 41 ++++++++---- .../fork/runner/listeners/RetryListener.java | 1 + .../listeners/TestRunListenersFactory.java | 7 +- 11 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 fork-runner/src/main/java/com/shazam/fork/model/Counter.java create mode 100644 fork-runner/src/main/java/com/shazam/fork/model/DeviceTestCaseAccumulator.java create mode 100644 fork-runner/src/main/java/com/shazam/fork/model/TestCaseEventCounter.java diff --git a/fork-runner/src/main/java/com/shazam/fork/model/Counter.java b/fork-runner/src/main/java/com/shazam/fork/model/Counter.java new file mode 100644 index 00000000..b973ed3c --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/Counter.java @@ -0,0 +1,47 @@ +package com.shazam.fork.model; + +import com.google.common.base.Objects; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Generci class to associate an object to a count. + * + * @param + */ +public class Counter { + + private T type; + private AtomicInteger count; + + public Counter(T type, int initialCount) { + this.type = type; + this.count = new AtomicInteger(initialCount); + } + + public int increaseCount(){ + return count.incrementAndGet(); + } + + public T getType() { + return type; + } + + public int getCount() { + return count.get(); + } + + @Override + public int hashCode() { + return type.hashCode(); + } + + @Override + public boolean equals(Object obj) { + + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + final Counter other = (Counter) obj; + return Objects.equal(this.type, other.type); + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/model/DeviceTestCaseAccumulator.java b/fork-runner/src/main/java/com/shazam/fork/model/DeviceTestCaseAccumulator.java new file mode 100644 index 00000000..1b1558ce --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/DeviceTestCaseAccumulator.java @@ -0,0 +1,64 @@ +package com.shazam.fork.model; + +import com.google.common.base.Predicate; +import com.google.common.base.Supplier; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.collect.FluentIterable.from; + +/** + * Class that keeps track of the number of times each testCase is executed for device. + */ +public class DeviceTestCaseAccumulator { + private static final Logger logger = LoggerFactory.getLogger(DeviceTestCaseAccumulator.class); + + private Multimap map = ArrayListMultimap.create(); + + public void record(Device device, TestCaseEvent testCaseEvent) { + + if (!map.containsKey(device)) { + map.put(device, createNew(testCaseEvent).get()); + } + + if (!from(map.get(device)).anyMatch(isSameTestCase(testCaseEvent))) { + TestCaseEventCounter aNew = createNew(testCaseEvent).get(); + aNew.increaseCount(); + map.get(device).add(aNew); + } else { + from(map.get(device)).firstMatch(isSameTestCase(testCaseEvent)).get().increaseCount(); + } + } + + public int getCount(Device device, TestCaseEvent testCaseEvent) { + if (map.containsKey(device)) { + return from(map.get(device)) + .firstMatch(isSameTestCase(testCaseEvent)).or(TestCaseEventCounter.EMPTY) + .getCount(); + } else { + return 0; + } + } + + private static Supplier createNew(final TestCaseEvent testCaseEvent) { + return new Supplier() { + @Override + public TestCaseEventCounter get() { + return new TestCaseEventCounter(testCaseEvent, 0); + } + }; + } + + private static Predicate isSameTestCase(final TestCaseEvent testCaseEvent) { + return new Predicate() { + @Override + public boolean apply(TestCaseEventCounter input) { + return input.getType() != null + && testCaseEvent.equals(input.getType()); + } + }; + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java index 483170aa..e37ce5b5 100644 --- a/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java +++ b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEvent.java @@ -1,5 +1,7 @@ package com.shazam.fork.model; +import com.google.common.base.Objects; + public class TestCaseEvent { private final String testMethod; @@ -23,4 +25,20 @@ public String getTestClass() { public boolean isIgnored() { return isIgnored; } + + @Override + public int hashCode() { + return Objects.hashCode( this.testMethod, this.testClass); + } + + @Override + public boolean equals(Object obj) { + + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + final TestCaseEvent other = (TestCaseEvent) obj; + return Objects.equal(this.testMethod, other.testMethod) + && Objects.equal(this.testClass, other.testClass); + } + } diff --git a/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEventCounter.java b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEventCounter.java new file mode 100644 index 00000000..fc00c86e --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/model/TestCaseEventCounter.java @@ -0,0 +1,11 @@ +package com.shazam.fork.model; + +public class TestCaseEventCounter extends Counter { + + public static final TestCaseEventCounter EMPTY = new TestCaseEventCounter(null, 0); + + public TestCaseEventCounter(TestCaseEvent type, int initialCount) { + super(type, initialCount); + } + +} diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java index b7ccdbd7..cd43d2c2 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java @@ -10,7 +10,10 @@ package com.shazam.fork.runner; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.DeviceTestCaseAccumulator; import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import java.util.HashMap; import java.util.Map; @@ -27,6 +30,7 @@ public class OverallProgressReporter implements ProgressReporter { private long endOfTests; private final Map poolProgressTrackers = new HashMap<>(); private final RetryWatchdog retryWatchdog = new RetryWatchdog(); + private final DeviceTestCaseAccumulator failedTestCasesAccumulator = new DeviceTestCaseAccumulator(); @Override public void start() { @@ -82,4 +86,13 @@ public RetryWatchdog retryWatchdog() { return retryWatchdog; } + @Override + public void recordFailedTestCase(Device device, TestCaseEvent testCase) { + failedTestCasesAccumulator.record(device, testCase); + } + + @Override + public int getTestFailuresCountPerDevice(Device device, TestCaseEvent testCase) { + return failedTestCasesAccumulator.getCount(device, testCase); + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java index d0521092..e704b20d 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunner.java @@ -100,7 +100,7 @@ private void failAnyDroppedClasses(Pool pool, Queue testClassQueue) { String poolName = pool.getName(); Device failedTestsDevice = aDevice().withSerial(DROPPED_BY + poolName).build(); ForkXmlTestRunListener xmlGenerator = getForkXmlTestRunListener(fileManager, configuration.getOutput(), - pool, failedTestsDevice, null/*nextTest*/); //TODO + pool, failedTestsDevice, null/*nextTest*/, progressReporter); //TODO Collection methods = nextTest.getUnignoredMethods(); xmlGenerator.testRunStarted(poolName, methods.size()); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java index bc197139..3fdcd4f1 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/ProgressReporter.java @@ -10,7 +10,9 @@ package com.shazam.fork.runner; +import com.shazam.fork.model.Device; import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; public interface ProgressReporter { @@ -34,4 +36,8 @@ public interface ProgressReporter { float getProgress(); RetryWatchdog retryWatchdog(); + + void recordFailedTestCase(Device device, TestCaseEvent testCase); + + int getTestFailuresCountPerDevice(Device device, TestCaseEvent testCase); } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java b/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java index e84d9c28..e9bafcf9 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java @@ -4,7 +4,7 @@ public class RetryWatchdog { - private AtomicInteger counter = new AtomicInteger(3); + private AtomicInteger counter = new AtomicInteger(6); public boolean allowRetry() { return counter.get() >= 0 && counter.getAndDecrement() > 0; diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java index 055dc507..cea234f5 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java @@ -18,6 +18,7 @@ import com.shazam.fork.model.Device; import com.shazam.fork.model.Pool; import com.shazam.fork.model.TestCaseEvent; +import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.system.io.FileManager; import com.shazam.fork.system.io.FileType; @@ -26,21 +27,27 @@ import javax.annotation.Nonnull; -import static java.lang.Boolean.FALSE; -import static java.lang.Boolean.TRUE; - public class ForkXmlTestRunListener extends XmlTestRunListener { private final FileManager fileManager; private final Pool pool; private final Device device; private final TestCaseEvent testCase; - private @Nonnull Boolean failedTest = FALSE; + @Nonnull + private + ProgressReporter progressReporter; + + TestIdentifier test; - public ForkXmlTestRunListener(FileManager fileManager, Pool pool, Device device, TestCaseEvent testCase) { + public ForkXmlTestRunListener(FileManager fileManager, + Pool pool, + Device device, + TestCaseEvent testCase, + @Nonnull ProgressReporter progressReporter) { this.fileManager = fileManager; this.pool = pool; this.device = device; this.testCase = testCase; + this.progressReporter = progressReporter; } @Override @@ -49,18 +56,26 @@ protected File getResultFile(File reportDir) { } @Override - public void testFailed(TestIdentifier test, String trace) { - this.failedTest = TRUE; - super.testFailed(test, trace); + public void testStarted(TestIdentifier test) { + this.test = test; + super.testStarted(test); } @Override protected Map getPropertiesAttributes() { - Map superAttribs = super.getPropertiesAttributes(); - return ImmutableMap.builder() - .putAll(superAttribs) - .put("hasFailed", failedTest.toString()) - .build(); + + ImmutableMap.Builder mapBuilder = ImmutableMap.builder() + .putAll(super.getPropertiesAttributes()); + + int testFailuresCountPerDevice = progressReporter.getTestFailuresCountPerDevice(device, new TestCaseEvent(test.getTestName(), test.getClassName(), false)); + if (testFailuresCountPerDevice > 0) + + mapBuilder + .put("previousFailureCounts", Integer.toString(testFailuresCountPerDevice)) + .build(); + + + return mapBuilder.build(); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java index ab442169..b249aea8 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/RetryListener.java @@ -55,6 +55,7 @@ public RetryListener(@Nonnull Device device, @Override public void testFailed(TestIdentifier test, String trace) { failedTests.add(test); + progressReporter.recordFailedTestCase(device, new TestCaseEvent(test.getTestName(), test.getClassName(), false)); if (progressReporter.retryWatchdog().allowRetry()) { queueOfTestsInPool.add(currentTestCaseEvent); logger.info("Test " + test.toString() + " enqueued again into device." + device.getSerial()); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java index 8d1ce3ab..d63ead52 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/TestRunListenersFactory.java @@ -50,7 +50,7 @@ public List createTestListeners(TestCaseEvent testCase, TestCaseEvent currentTestCaseEvent) { return asList( new ProgressTestRunListener(pool, progressReporter), - getForkXmlTestRunListener(fileManager, configuration.getOutput(), pool, device, testCase), + getForkXmlTestRunListener(fileManager, configuration.getOutput(), pool, device, testCase, progressReporter), new ConsoleLoggingTestRunListener(configuration.getTestPackage(), device.getSerial(), device.getModelName(), progressReporter), new LogCatTestRunListener(gson, fileManager, pool, device), @@ -63,8 +63,9 @@ public static ForkXmlTestRunListener getForkXmlTestRunListener(FileManager fileM File output, Pool pool, Device device, - TestCaseEvent testCase) { - ForkXmlTestRunListener xmlTestRunListener = new ForkXmlTestRunListener(fileManager, pool, device, testCase); + TestCaseEvent testCase, + ProgressReporter progressReporter) { + ForkXmlTestRunListener xmlTestRunListener = new ForkXmlTestRunListener(fileManager, pool, device, testCase, progressReporter); xmlTestRunListener.setReportDir(output); return xmlTestRunListener; } From 1f8b648577559f501eadabc9b978d04c67e97053 Mon Sep 17 00:00:00 2001 From: Francesco Pezzato Date: Tue, 8 Mar 2016 11:33:37 +0000 Subject: [PATCH 04/22] Add totalAllowedRetryQuota CLI param. --- .../java/com/shazam/fork/Configuration.java | 16 +++++++++- .../java/com/shazam/fork/ForkBuilder.java | 31 +++++++++++++++++-- .../main/java/com/shazam/fork/ForkCli.java | 12 +++++++ .../fork/injector/RetryWatchdogInjector.java | 12 +++++++ .../runner/ProgressReporterInjector.java | 6 ++-- .../fork/runner/OverallProgressReporter.java | 6 +++- .../fork/runner/PoolTestRunnerFactory.java | 1 + .../com/shazam/fork/runner/RetryWatchdog.java | 10 ++++-- .../listeners/ForkXmlTestRunListener.java | 2 +- 9 files changed, 87 insertions(+), 9 deletions(-) create mode 100644 fork-runner/src/main/java/com/shazam/fork/injector/RetryWatchdogInjector.java diff --git a/fork-runner/src/main/java/com/shazam/fork/Configuration.java b/fork-runner/src/main/java/com/shazam/fork/Configuration.java index 7d4b3aac..75abccce 100644 --- a/fork-runner/src/main/java/com/shazam/fork/Configuration.java +++ b/fork-runner/src/main/java/com/shazam/fork/Configuration.java @@ -30,6 +30,8 @@ public class Configuration { private final String testPackage; private final int testOutputTimeout; private final boolean fallbackToScreenshots; + private final int totalAllowedRetryQuota; + private final int retryPerTestCaseQuota; public Configuration(@Nonnull File androidSdk, @Nonnull File applicationApk, @@ -40,7 +42,9 @@ public Configuration(@Nonnull File androidSdk, @Nonnull Pattern testPackagePattern, @Nonnull String testPackage, int testOutputTimeout, - boolean fallbackToScreenshots) { + boolean fallbackToScreenshots, + int totalAllowedRetryQuota, + int retryPerTestCaseQuota) { this.androidSdk = androidSdk; this.applicationApk = applicationApk; this.instrumentationApk = instrumentationApk; @@ -51,6 +55,8 @@ public Configuration(@Nonnull File androidSdk, this.testPackage = testPackage; this.testOutputTimeout = testOutputTimeout; this.fallbackToScreenshots = fallbackToScreenshots; + this.totalAllowedRetryQuota = totalAllowedRetryQuota; + this.retryPerTestCaseQuota = retryPerTestCaseQuota; } @Nonnull @@ -99,4 +105,12 @@ public int getTestOutputTimeout() { public boolean canFallbackToScreenshots() { return fallbackToScreenshots; } + + public int getTotalAllowedRetryQuota() { + return totalAllowedRetryQuota; + } + + public int getRetryPerTestCaseQuota() { + return retryPerTestCaseQuota; + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java b/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java index 1ce57bd9..d6b41bea 100644 --- a/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java +++ b/fork-runner/src/main/java/com/shazam/fork/ForkBuilder.java @@ -30,6 +30,8 @@ public class ForkBuilder { private String testPackage; // Will default to test APK's package name as soon as it's known private int testOutputTimeout = Defaults.TEST_OUTPUT_TIMEOUT_MILLIS; private boolean fallbackToScreenshots = true; + private int totalAllowedRetryQuota = 0; + private int retryPerTestCaseQuota = 1; public static ForkBuilder aFork() { return new ForkBuilder(); @@ -116,6 +118,29 @@ public ForkBuilder withFallbackToScreenshots(boolean fallbackToScreenshots) { return this; } + /** + * Allow to re-run tests. + * + * @param totalAllowedRetryQuota Threshold of total failures below whom the failed tests can be executed again. + * @return this builder + */ + public ForkBuilder withTotalAllowedRetryQuota(int totalAllowedRetryQuota) { + this.totalAllowedRetryQuota = totalAllowedRetryQuota; + return this; + } + + /** + * Max number of time each testCase is attempted again. + * @param retryPerTestCaseQuota max number of attempts when rerunning tests. + * @return this builder. + */ + //TODO - implement. + public ForkBuilder withRetryPerTestCaseQuota(int retryPerTestCaseQuota) { + this.retryPerTestCaseQuota = retryPerTestCaseQuota; + return this; + } + + public Fork build() { checkNotNull(androidSdk, "SDK is required."); checkArgument(androidSdk.exists(), "SDK directory does not exist."); @@ -125,6 +150,7 @@ public Fork build() { checkArgument(instrumentationApk.exists(), "Instrumentation APK file does not exist."); checkNotNull(output, "Output path is required."); checkArgument(testOutputTimeout >= 0, "Timeout must be non-negative."); + checkArgument(totalAllowedRetryQuota >= 0, "Total allowed retry quota should not be negative."); InstrumentationInfo instrumentationInfo = parseFromFile(instrumentationApk); String testPackage = configuredOrInstrumentationPackage(instrumentationInfo.getInstrumentationPackage()); @@ -138,8 +164,9 @@ public Fork build() { compilePatternFor(testPackage), testPackage, testOutputTimeout, - fallbackToScreenshots - ); + fallbackToScreenshots, + totalAllowedRetryQuota, + retryPerTestCaseQuota); return new Fork(configuration); } diff --git a/fork-runner/src/main/java/com/shazam/fork/ForkCli.java b/fork-runner/src/main/java/com/shazam/fork/ForkCli.java index 3a26bd79..cd363364 100644 --- a/fork-runner/src/main/java/com/shazam/fork/ForkCli.java +++ b/fork-runner/src/main/java/com/shazam/fork/ForkCli.java @@ -63,6 +63,14 @@ public static class CommandLineArgs { @Parameter(names = { "-h", "--help" }, description = "Command help", help = true, hidden = true) public boolean help; + + @Parameter(names = { "--total-allowed-retry-quota" }, description = "Amount of re-executions of failing tests allowed.", converter = IntegerConverter.class) + public int totalAllowedRetryQuota = 0; + + //TODO - enable when finished. + /* @Parameter(names = { "--retry-per-test-case-quota" }, description = "Max number of time each testCase is attempted again " + + "before declaring it as a failure.", converter = IntegerConverter.class) + public int retryPerTestCaseQuota = 1;*/ } /* JCommander deems it necessary that this class be public. Lame. */ @@ -125,5 +133,9 @@ private static void overrideDefaultsIfSet(ForkBuilder forkBuilder, CommandLineAr if (parsedArgs.testOutputTimeout > -1) { forkBuilder.withTestOutputTimeout(parsedArgs.testOutputTimeout); } + + if (parsedArgs.totalAllowedRetryQuota > 0) { + forkBuilder.withTotalAllowedRetryQuota(parsedArgs.totalAllowedRetryQuota); + } } } diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/RetryWatchdogInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/RetryWatchdogInjector.java new file mode 100644 index 00000000..4b1439f2 --- /dev/null +++ b/fork-runner/src/main/java/com/shazam/fork/injector/RetryWatchdogInjector.java @@ -0,0 +1,12 @@ +package com.shazam.fork.injector; + +import com.shazam.fork.runner.RetryWatchdog; + +import static com.shazam.fork.injector.ConfigurationInjector.configuration; + +public class RetryWatchdogInjector { + + public static RetryWatchdog retryWatchdog(){ + return new RetryWatchdog(configuration()); + } +} diff --git a/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java b/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java index 155e1913..8924b189 100644 --- a/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java +++ b/fork-runner/src/main/java/com/shazam/fork/injector/runner/ProgressReporterInjector.java @@ -10,12 +10,14 @@ package com.shazam.fork.injector.runner; -import com.shazam.fork.runner.ProgressReporter; import com.shazam.fork.runner.OverallProgressReporter; +import com.shazam.fork.runner.ProgressReporter; + +import static com.shazam.fork.injector.RetryWatchdogInjector.retryWatchdog; public class ProgressReporterInjector { public static ProgressReporter progressReporter() { - return new OverallProgressReporter(); + return new OverallProgressReporter(retryWatchdog()); } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java index cd43d2c2..3224f013 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/OverallProgressReporter.java @@ -29,9 +29,13 @@ public class OverallProgressReporter implements ProgressReporter { private long startOfTests; private long endOfTests; private final Map poolProgressTrackers = new HashMap<>(); - private final RetryWatchdog retryWatchdog = new RetryWatchdog(); + private final RetryWatchdog retryWatchdog; private final DeviceTestCaseAccumulator failedTestCasesAccumulator = new DeviceTestCaseAccumulator(); + public OverallProgressReporter(RetryWatchdog retryWatchdog) { + this.retryWatchdog = retryWatchdog; + } + @Override public void start() { startOfTests = nanoTime(); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java index d8f9c7ba..fed21d54 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/PoolTestRunnerFactory.java @@ -52,6 +52,7 @@ public PoolTestRunner createPoolTestRunner(Pool pool, failureAccumulator); } + //TODO cleanup. /* private int countTests(List testClasses) { int sum = 0; for (TestClass testClass : testClasses) { diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java b/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java index e9bafcf9..bceac6dd 100644 --- a/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/RetryWatchdog.java @@ -1,12 +1,18 @@ package com.shazam.fork.runner; +import com.shazam.fork.Configuration; + import java.util.concurrent.atomic.AtomicInteger; public class RetryWatchdog { - private AtomicInteger counter = new AtomicInteger(6); + private AtomicInteger totalAllowedRetryQuota; + + public RetryWatchdog(Configuration configuration) { + totalAllowedRetryQuota = new AtomicInteger(configuration.getTotalAllowedRetryQuota()); + } public boolean allowRetry() { - return counter.get() >= 0 && counter.getAndDecrement() > 0; + return totalAllowedRetryQuota.get() > 0 && totalAllowedRetryQuota.getAndDecrement() >= 0; } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java index cea234f5..0697dd5a 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java @@ -71,7 +71,7 @@ protected Map getPropertiesAttributes() { if (testFailuresCountPerDevice > 0) mapBuilder - .put("previousFailureCounts", Integer.toString(testFailuresCountPerDevice)) + .put("totalFailureCount", Integer.toString(testFailuresCountPerDevice)) .build(); From f6d5321bbfe15b97154f66726f09720c197ad462 Mon Sep 17 00:00:00 2001 From: Francesco Pezzato Date: Tue, 8 Mar 2016 11:33:52 +0000 Subject: [PATCH 05/22] Ensure ForkXmlTestRunListener handles empty tests. --- .../runner/listeners/ForkXmlTestRunListener.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java index 0697dd5a..01ba8095 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java @@ -66,16 +66,17 @@ protected Map getPropertiesAttributes() { ImmutableMap.Builder mapBuilder = ImmutableMap.builder() .putAll(super.getPropertiesAttributes()); + if(test != null) { - int testFailuresCountPerDevice = progressReporter.getTestFailuresCountPerDevice(device, new TestCaseEvent(test.getTestName(), test.getClassName(), false)); - if (testFailuresCountPerDevice > 0) - - mapBuilder - .put("totalFailureCount", Integer.toString(testFailuresCountPerDevice)) - .build(); + int testFailuresCountPerDevice = progressReporter.getTestFailuresCountPerDevice(device, + new TestCaseEvent(test.getTestName(), test.getClassName(), false)); + if (testFailuresCountPerDevice > 0) + mapBuilder + .put("totalFailureCount", Integer.toString(testFailuresCountPerDevice)) + .build(); + } return mapBuilder.build(); - } } From b4ab968ac74b0d8a8c8ce202b755e7668931105d Mon Sep 17 00:00:00 2001 From: Francesco Pezzato Date: Tue, 8 Mar 2016 11:34:10 +0000 Subject: [PATCH 06/22] Print text summary of failed tests. --- .../java/com/shazam/fork/summary/Summary.java | 12 ++ .../com/shazam/fork/summary/TestResult.java | 16 +++ .../listeners/ForkXmlTestRunListener.java | 14 +- .../shazam/fork/summary/HtmlConverters.java | 1 + .../com/shazam/fork/summary/HtmlSummary.java | 1 + .../fork/summary/OutcomeAggregator.java | 2 +- .../shazam/fork/summary/SummaryCompiler.java | 120 ++++++++++-------- .../com/shazam/fork/summary/TestSuite.java | 17 ++- .../src/main/resources/forkpages/index.html | 10 ++ 9 files changed, 131 insertions(+), 62 deletions(-) diff --git a/fork-common/src/main/java/com/shazam/fork/summary/Summary.java b/fork-common/src/main/java/com/shazam/fork/summary/Summary.java index 423a0011..4905a38e 100755 --- a/fork-common/src/main/java/com/shazam/fork/summary/Summary.java +++ b/fork-common/src/main/java/com/shazam/fork/summary/Summary.java @@ -23,6 +23,7 @@ public class Summary { private final String title; private final String subtitle; private final ArrayList ignoredTests; + private final ArrayList failedTests; @Nonnull public List getPoolSummaries() { @@ -42,11 +43,16 @@ public ArrayList getIgnoredTests() { return ignoredTests; } + public ArrayList getFailedTests() { + return failedTests; + } + public static class Builder { private final List poolSummaries = new ArrayList<>(); private final ArrayList ignoredTests = new ArrayList<>(); private String title = "Report Title"; private String subtitle = "Report Subtitle"; + private ArrayList failedTests = new ArrayList<>(); public static Builder aSummary() { return new Builder(); @@ -72,6 +78,11 @@ public Builder addIgnoredTest(String s) { return this; } + public Builder addFailedTests(String failedTests) { + this.failedTests.add(failedTests); + return this; + } + public Summary build() { return new Summary(this); } @@ -82,5 +93,6 @@ private Summary(Builder builder) { title = builder.title; subtitle = builder.subtitle; ignoredTests = builder.ignoredTests; + this.failedTests = builder.failedTests; } } diff --git a/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java b/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java index ae697173..9c82a902 100755 --- a/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java +++ b/fork-common/src/main/java/com/shazam/fork/summary/TestResult.java @@ -14,6 +14,9 @@ import com.shazam.fork.model.Device; +import java.util.HashMap; +import java.util.Map; + import javax.annotation.Nonnull; import static com.google.common.base.Strings.isNullOrEmpty; @@ -25,6 +28,7 @@ public class TestResult { private final String testMethod; private final String errorTrace; private final String failureTrace; + private final Map testMetrics; public Device getDevice() { return device; @@ -42,6 +46,10 @@ public String getTestMethod() { return testMethod; } + public Map getTestMetrics() { + return testMetrics; + } + @Nonnull public ResultStatus getResultStatus() { if (!isNullOrEmpty(errorTrace)) { @@ -71,6 +79,7 @@ public static class Builder { private String testMethod; private String errorTrace; private String failureTrace; + private Map testMetrics = new HashMap(); public static Builder aTestResult() { return new Builder(); @@ -110,6 +119,12 @@ public Builder withFailureTrace(String trace) { return this; } + public Builder withTestMetrics(Map testMetrics) { + this.testMetrics.clear(); + this.testMetrics.putAll(testMetrics); + return this; + } + public TestResult build() { return new TestResult(this); } @@ -123,5 +138,6 @@ private TestResult(Builder builder) { testMethod = builder.testMethod; errorTrace = builder.errorTrace; failureTrace = builder.failureTrace; + this.testMetrics = builder.testMetrics; } } diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java index 01ba8095..917fad8e 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/listeners/ForkXmlTestRunListener.java @@ -28,14 +28,16 @@ import javax.annotation.Nonnull; public class ForkXmlTestRunListener extends XmlTestRunListener { + public static final String SUMMARY_KEY_TOTAL_FAILURE_COUNT = "totalFailureCount"; + private final FileManager fileManager; private final Pool pool; private final Device device; private final TestCaseEvent testCase; + @Nonnull private ProgressReporter progressReporter; - TestIdentifier test; public ForkXmlTestRunListener(FileManager fileManager, @@ -66,16 +68,14 @@ protected Map getPropertiesAttributes() { ImmutableMap.Builder mapBuilder = ImmutableMap.builder() .putAll(super.getPropertiesAttributes()); - if(test != null) { - + if (test != null) { int testFailuresCountPerDevice = progressReporter.getTestFailuresCountPerDevice(device, new TestCaseEvent(test.getTestName(), test.getClassName(), false)); - if (testFailuresCountPerDevice > 0) - + if (testFailuresCountPerDevice > 0) { mapBuilder - .put("totalFailureCount", Integer.toString(testFailuresCountPerDevice)) + .put(SUMMARY_KEY_TOTAL_FAILURE_COUNT, Integer.toString(testFailuresCountPerDevice)) .build(); - + } } return mapBuilder.build(); } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java index 9d1842fd..d63ccc07 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlConverters.java @@ -35,6 +35,7 @@ public static HtmlSummary toHtmlSummary(Summary summary) { htmlSummary.subtitle = summary.getSubtitle(); htmlSummary.pools = transform(summary.getPoolSummaries(), toHtmlPoolSummary()); htmlSummary.ignoredTests = summary.getIgnoredTests(); + htmlSummary.failedTests = summary.getFailedTests(); htmlSummary.overallStatus = new OutcomeAggregator().aggregate(summary) ? "pass" : "fail"; return htmlSummary; } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java index bc6fff30..9dca15eb 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/HtmlSummary.java @@ -24,4 +24,5 @@ public class HtmlSummary { public String subtitle; public ArrayList ignoredTests; public String overallStatus; + public ArrayList failedTests; } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java b/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java index fb8271d5..9eb64d82 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/OutcomeAggregator.java @@ -43,7 +43,7 @@ public static Function toPoolOutcome() { public Boolean apply(@Nullable PoolSummary input) { final Collection testResults = input.getTestResults(); final Collection testOutcomes = transform(testResults, toTestOutcome()); - return and(testOutcomes); + return testOutcomes.size() > 0 && and(testOutcomes); } }; } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java index 381c45d6..8b6d588c 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/SummaryCompiler.java @@ -12,9 +12,11 @@ */ package com.shazam.fork.summary; -import com.google.common.base.Function; +import com.google.common.collect.Lists; import com.shazam.fork.RuntimeConfiguration; -import com.shazam.fork.model.*; +import com.shazam.fork.model.Device; +import com.shazam.fork.model.Pool; +import com.shazam.fork.model.TestCaseEvent; import com.shazam.fork.runner.PoolTestRunner; import com.shazam.fork.system.io.FileManager; @@ -22,12 +24,12 @@ import org.simpleframework.xml.core.Persister; import java.io.File; -import java.util.*; +import java.util.Collection; +import java.util.List; +import java.util.Map; -import javax.annotation.Nonnull; - -import static com.google.common.collect.Collections2.transform; import static com.shazam.fork.model.Device.Builder.aDevice; +import static com.shazam.fork.runner.listeners.ForkXmlTestRunListener.SUMMARY_KEY_TOTAL_FAILURE_COUNT; import static com.shazam.fork.summary.PoolSummary.Builder.aPoolSummary; import static com.shazam.fork.summary.Summary.Builder.aSummary; import static com.shazam.fork.summary.TestResult.Builder.aTestResult; @@ -43,40 +45,41 @@ public class SummaryCompiler { public SummaryCompiler(RuntimeConfiguration runtimeConfiguration, FileManager fileManager) { this.runtimeConfiguration = runtimeConfiguration; this.fileManager = fileManager; - serializer = new Persister(); - } + serializer = new Persister(); + } - Summary compileSummary(Collection pools, List testCases) { - Summary.Builder summaryBuilder = aSummary(); - for (Pool pool : pools) { - PoolSummary poolSummary = compilePoolSummary(pool); + Summary compileSummary(Collection pools, List testCases) { + Summary.Builder summaryBuilder = aSummary(); + for (Pool pool : pools) { + PoolSummary poolSummary = compilePoolSummary(pool, summaryBuilder); summaryBuilder.addPoolSummary(poolSummary); - } + } addIgnoredTests(testCases, summaryBuilder); summaryBuilder.withTitle(runtimeConfiguration.getTitle()); summaryBuilder.withSubtitle(runtimeConfiguration.getSubtitle()); - return summaryBuilder.build(); - } + return summaryBuilder.build(); + } - private PoolSummary compilePoolSummary(Pool pool) { + private PoolSummary compilePoolSummary(Pool pool, Summary.Builder summaryBuilder) { PoolSummary.Builder poolSummaryBuilder = aPoolSummary().withPoolName(pool.getName()); - for (Device device: pool.getDevices()) { - compileResultsForDevice(pool, poolSummaryBuilder, device); + for (Device device : pool.getDevices()) { + compileResultsForDevice(pool, poolSummaryBuilder, summaryBuilder, device); } Device watchdog = getPoolWatchdog(pool.getName()); - compileResultsForDevice(pool, poolSummaryBuilder, watchdog); + compileResultsForDevice(pool, poolSummaryBuilder, summaryBuilder, watchdog); return poolSummaryBuilder.build(); } - private void compileResultsForDevice(Pool pool, PoolSummary.Builder poolSummaryBuilder, Device device) { + private void compileResultsForDevice(Pool pool, PoolSummary.Builder poolSummaryBuilder, Summary.Builder summaryBuilder, Device device) { File[] deviceResultFiles = fileManager.getTestFilesForDevice(pool, device); if (deviceResultFiles == null) { return; } for (File file : deviceResultFiles) { - Collection testResults = parseTestResultsFromFile(file, device); - poolSummaryBuilder.addTestResults(testResults); + Collection testResult = parseTestResultsFromFile(file, device); + poolSummaryBuilder.addTestResults(testResult); + addFailedTests(testResult, summaryBuilder); } } @@ -96,33 +99,48 @@ private void addIgnoredTests(List testCases, Summary.Builder summ } } - private Collection parseTestResultsFromFile(File file, Device device) { - try { - TestSuite testSuite = serializer.read(TestSuite.class, file, STRICT); - List testCases = testSuite.getTestCases(); - if ((testCases == null) || testCases.isEmpty()) { - return new ArrayList<>(0); - } - return transform(testCases, toTestResult(device)); - } catch (Exception e) { - throw new RuntimeException("Error when parsing file: " + file.getAbsolutePath(), e); - } - } - - private Function toTestResult(final Device device) { - return new Function() { - @SuppressWarnings("NullableProblems") - @Override - public TestResult apply(@Nonnull TestCase testCase) { - return aTestResult() - .withDevice(device) - .withTestClass(testCase.getClassname()) - .withTestMethod(testCase.getName()) - .withTimeTaken(testCase.getTime()) - .withErrorTrace(testCase.getError()) - .withFailureTrace(testCase.getFailure()) - .build(); - } - }; - } + private void addFailedTests(Collection testResults, Summary.Builder summaryBuilder) { + for (TestResult testResult : testResults) { + Map testMetrics = testResult.getTestMetrics(); + if (testMetrics != null + && testMetrics.containsKey(SUMMARY_KEY_TOTAL_FAILURE_COUNT)) { + String failedTest = testMetrics.get(SUMMARY_KEY_TOTAL_FAILURE_COUNT) + + " times " + testResult.getTestClass() + "#" + testResult.getTestMethod() + " on " + testResult.getDevice().getSerial() ; + summaryBuilder.addFailedTests(failedTest); + } + } + } + + private Collection parseTestResultsFromFile(File file, Device device) { + try { + TestSuite testSuite = serializer.read(TestSuite.class, file, STRICT); + Collection testCases = testSuite.getTestCase(); + List result = Lists.newArrayList(); + if ((testCases == null)) { + return result; + } + + for(TestCase testCase : testCases){ + TestResult testResult = getTestResult(device, testSuite, testCase); + result.add(testResult); + } + return result; + } catch (Exception e) { + throw new RuntimeException("Error when parsing file: " + file.getAbsolutePath(), e); + } + } + + private TestResult getTestResult(Device device, TestSuite testSuite, TestCase testCase) { + TestResult.Builder testResultBuilder = aTestResult() + .withDevice(device) + .withTestClass(testCase.getClassname()) + .withTestMethod(testCase.getName()) + .withTimeTaken(testCase.getTime()) + .withErrorTrace(testCase.getError()) + .withFailureTrace(testCase.getFailure()); + if (testSuite.getProperties() != null) { + testResultBuilder.withTestMetrics(testSuite.getProperties()); + } + return testResultBuilder.build(); + } } diff --git a/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java b/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java index 915ab5be..ecd22bc1 100755 --- a/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java +++ b/fork-runner/src/main/java/com/shazam/fork/summary/TestSuite.java @@ -13,9 +13,12 @@ package com.shazam.fork.summary; import org.simpleframework.xml.ElementList; +import org.simpleframework.xml.ElementMap; +import org.simpleframework.xml.Path; import org.simpleframework.xml.Root; import java.util.List; +import java.util.Map; @Root @@ -24,7 +27,15 @@ class TestSuite { @ElementList(inline=true, type=TestCase.class, required=false) private List testCases; - public List getTestCases() { - return testCases; - } + @Path("./properties") + @ElementMap(required = false, entry = "property", key = "name", value = "value", attribute = true, inline = true) + private Map properties; + + public List getTestCase() { + return testCases; + } + + public Map getProperties() { + return properties; + } } diff --git a/fork-runner/src/main/resources/forkpages/index.html b/fork-runner/src/main/resources/forkpages/index.html index 23086f78..5d1c7b5e 100755 --- a/fork-runner/src/main/resources/forkpages/index.html +++ b/fork-runner/src/main/resources/forkpages/index.html @@ -52,6 +52,16 @@

{{title}}

{{/ignoredTests}} + Failed tests: +
    + {{#failedTests}} +
  • {{.}}
    • + {{/failedTests}} + {{^failedTests}} +
  • None.
    • + {{/failedTests}} +
+