diff --git a/fork-common-test-dex/app/src/androidTest/java/com/shazam/forktest/ParameterizedClassTest.java b/fork-common-test-dex/app/src/androidTest/java/com/shazam/forktest/ParameterizedClassTest.java new file mode 100644 index 00000000..8fcaf35e --- /dev/null +++ b/fork-common-test-dex/app/src/androidTest/java/com/shazam/forktest/ParameterizedClassTest.java @@ -0,0 +1,33 @@ +package com.shazam.forktest; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import com.shazam.forktest.ExcludedAnnotation; + +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) +public class ParameterizedClassTest { + private String id; + private boolean flag; + + @Parameterized.Parameters(name = "{0}({1})") + public static Collection params() { + return Arrays.asList(new Object[][] { + { "first", false }, + { "second", true }, + { "third", false }, + }); + } + + @Test + public void test() { + assertEquals(flag, ("second".equals(id))); + } +} diff --git a/fork-common/src/main/java/com/shazam/fork/model/TestCaseEvent.java b/fork-common/src/main/java/com/shazam/fork/model/TestCaseEvent.java index ca8becea..5a3e3067 100644 --- a/fork-common/src/main/java/com/shazam/fork/model/TestCaseEvent.java +++ b/fork-common/src/main/java/com/shazam/fork/model/TestCaseEvent.java @@ -26,12 +26,15 @@ public class TestCaseEvent { @Nonnull private final Map properties; + private final boolean isParameterized; + private TestCaseEvent(Builder builder) { this.testClass = builder.testClass; this.testMethod = builder.testMethod; this.isIgnored = builder.isIgnored; this.permissionsToRevoke = builder.permissionsToRevoke; this.properties = builder.properties; + this.isParameterized = builder.isParameterized; } @Nonnull @@ -68,6 +71,10 @@ public Map getProperties() { return unmodifiableMap(properties); } + public boolean isParameterized() { + return isParameterized; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -103,6 +110,7 @@ public static class Builder { private boolean isIgnored; private List permissionsToRevoke = new ArrayList<>(); private Map properties = new HashMap<>(); + private boolean isParameterized = false; @Nonnull public static Builder testCaseEvent(@Nonnull TestIdentifier testIdentifier) { @@ -148,6 +156,12 @@ public Builder withProperties(@Nonnull Map properties) { return this; } + @Nonnull + public Builder withIsParameterized(boolean flag) { + this.isParameterized = flag; + return this; + } + @Nonnull public TestCaseEvent build() { return new TestCaseEvent(this); diff --git a/fork-common/src/main/java/com/shazam/fork/suite/TestSuiteLoader.java b/fork-common/src/main/java/com/shazam/fork/suite/TestSuiteLoader.java index 8f433152..19ffe389 100644 --- a/fork-common/src/main/java/com/shazam/fork/suite/TestSuiteLoader.java +++ b/fork-common/src/main/java/com/shazam/fork/suite/TestSuiteLoader.java @@ -13,6 +13,7 @@ import com.shazam.fork.io.DexFileExtractor; import com.shazam.fork.model.TestCaseEvent; +import com.shazam.fork.utils.ParameterizedTestDetector; import org.jf.dexlib2.iface.Annotation; import org.jf.dexlib2.iface.AnnotationElement; import org.jf.dexlib2.iface.ClassDef; @@ -28,6 +29,7 @@ import java.io.File; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,6 +39,7 @@ import static com.shazam.fork.model.TestCaseEvent.Builder.testCaseEvent; import static java.lang.Math.min; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; public class TestSuiteLoader { @@ -74,6 +77,13 @@ public Collection loadTestSuite() throws NoTestCasesFoundExceptio if (testCaseEvents.isEmpty()) { throw new NoTestCasesFoundException("No tests cases were found in the test APK: " + instrumentationApkFile.getAbsolutePath()); } + + // put parameterized tests to begin of queue to run them first + // because most probably they will require more time than any regular test + Map> testEventsByParametrizedFlag = testCaseEvents.stream() + .collect(groupingBy(TestCaseEvent::isParameterized, toList())); + testCaseEvents = testEventsByParametrizedFlag.getOrDefault(true, new ArrayList<>()); + testCaseEvents.addAll(testEventsByParametrizedFlag.getOrDefault(false, new ArrayList<>())); return testCaseEvents; } @@ -82,9 +92,23 @@ private List convertClassToTestCaseEvents(ClassDef classDefItem) List testCaseEvents = new ArrayList<>(); Iterable methods = classDefItem.getMethods(); - StreamSupport.stream(methods.spliterator(), false) - .filter(method -> method.getAnnotations().stream().anyMatch(annotation -> TEST_ANNOTATION.equals(annotation.getType()))) - .map(method -> convertToTestCaseEvent(classDefItem, method)).forEach(testCaseEvents::add); + if (ParameterizedTestDetector.isParameterizedClass(classDefItem)) { + return Collections.singletonList( + TestCaseEvent.Builder.testCaseEvent() + .withIsParameterized(true) + .withTestClass(getClassName(classDefItem)) + .withTestMethod("") + // we can't properly filter tests in parameterized classes + // because we invoke whole class, not methods + // so can check only marker for class + .withIsIgnored(isClassIgnored(classDefItem)) + .build() + ); + } else { + StreamSupport.stream(methods.spliterator(), false) + .filter(method -> method.getAnnotations().stream().anyMatch(annotation -> TEST_ANNOTATION.equals(annotation.getType()))) + .map(method -> convertToTestCaseEvent(classDefItem, method)).forEach(testCaseEvents::add); + } return testCaseEvents; } diff --git a/fork-common/src/main/java/com/shazam/fork/utils/ParameterizedTestDetector.java b/fork-common/src/main/java/com/shazam/fork/utils/ParameterizedTestDetector.java new file mode 100644 index 00000000..3c3548fe --- /dev/null +++ b/fork-common/src/main/java/com/shazam/fork/utils/ParameterizedTestDetector.java @@ -0,0 +1,31 @@ +package com.shazam.fork.utils; + +import org.jf.dexlib2.ValueType; +import org.jf.dexlib2.dexbacked.value.DexBackedTypeEncodedValue; +import org.jf.dexlib2.iface.ClassDef; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +public class ParameterizedTestDetector { + public static final String RUN_WITH_ANNOTATION = "Lorg/junit/runner/RunWith;"; + public static final Set PARAMETRIZED_ANNOTATIONS = new HashSet<>(Arrays.asList( + "Lorg/junit/runners/Parameterized;", + "Lio/qameta/allure/kotlin/junit4/AllureParametrizedRunner;" + )); + + public static boolean isParameterizedClass(ClassDef classDef) { + Optional encodedValue = classDef.getAnnotations().stream() + .filter(it -> RUN_WITH_ANNOTATION.equals(it.getType())) + .flatMap(it -> it.getElements().stream()) + .filter(it -> + it.getValue().getValueType() == ValueType.TYPE + && it.getValue() instanceof DexBackedTypeEncodedValue) + .findFirst() + .map(it -> (DexBackedTypeEncodedValue) it.getValue()); + return encodedValue.isPresent() + && PARAMETRIZED_ANNOTATIONS.contains(encodedValue.get().getValue()); + } +} diff --git a/fork-common/src/test/java/com/shazam/fork/suite/TestSuiteLoaderTest.java b/fork-common/src/test/java/com/shazam/fork/suite/TestSuiteLoaderTest.java index 8a671ab0..5421bb71 100644 --- a/fork-common/src/test/java/com/shazam/fork/suite/TestSuiteLoaderTest.java +++ b/fork-common/src/test/java/com/shazam/fork/suite/TestSuiteLoaderTest.java @@ -20,6 +20,7 @@ import javax.annotation.Nonnull; import java.io.File; import java.net.URL; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,6 +35,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.is; /** * This test is based on the tests.dex file, which contains test classes with the following code: @@ -160,6 +162,15 @@ public void populatesTestProperties() throws Exception { sameTestEventAs("methodWithUnmatchedKey", testClass, singlePropertyMap))); } + @SuppressWarnings("unchecked") + @Test + public void checkParameterizedTest() throws Exception { + Collection testCaseEvents = testSuiteLoader.loadTestSuite(); + assertThat(testCaseEvents.stream().filter(TestCaseEvent::isParameterized).count(), is(1L)); + assertThat(testCaseEvents, hasItems( + sameTestEventAs("com.shazam.forktest.ParameterizedClassTest", true))); + } + @Nonnull private Matcher sameTestEventAs(String testMethod, String testClass, Map properties) { return sameBeanAs( @@ -171,6 +182,16 @@ private Matcher sameTestEventAs(String testMethod, String testCla ); } + private Matcher sameTestEventAs(String testClass, boolean isParameterized) { + return sameBeanAs( + testCaseEvent() + .withTestClass(testClass) + .withTestMethod("") + .withIsParameterized(isParameterized) + .build() + ); + } + @Nonnull private Matcher sameTestEventAs(String testMethod, String testClass, boolean isIgnored) { return sameBeanAs( diff --git a/fork-common/src/test/resources/app-debug.apk b/fork-common/src/test/resources/app-debug.apk index 78d1b905..23449c56 100644 Binary files a/fork-common/src/test/resources/app-debug.apk and b/fork-common/src/test/resources/app-debug.apk differ diff --git a/fork-common/src/test/resources/tests.dex b/fork-common/src/test/resources/tests.dex index f5ed5031..f3f3da94 100644 Binary files a/fork-common/src/test/resources/tests.dex and b/fork-common/src/test/resources/tests.dex differ diff --git a/fork-runner/src/main/java/com/shazam/fork/aggregator/FilesRetrieverBasedAggregator.java b/fork-runner/src/main/java/com/shazam/fork/aggregator/FilesRetrieverBasedAggregator.java index 91fef851..794d7658 100644 --- a/fork-runner/src/main/java/com/shazam/fork/aggregator/FilesRetrieverBasedAggregator.java +++ b/fork-runner/src/main/java/com/shazam/fork/aggregator/FilesRetrieverBasedAggregator.java @@ -101,6 +101,7 @@ private static List getFatalCrashedTests(Collection proc .map(testResult -> new TestResultItem(testResult.getTestClass(), testResult.getTestMethod())) .collect(toSet()); Set allTests = testCases.stream() + .filter(it -> !it.isParameterized()) .map(testCaseEvent -> new TestResultItem(testCaseEvent.getTestClass(), testCaseEvent.getTestMethod())) .collect(toSet()); diff --git a/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java b/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java index 7ebb89c8..7fe374e8 100755 --- a/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java +++ b/fork-runner/src/main/java/com/shazam/fork/runner/TestRun.java @@ -64,7 +64,11 @@ public void execute() { runner.setTestSize(testSize); } runner.setRunName(poolName); - runner.setMethodName(testClassName, testMethodName); + if (test.isParameterized()) { + runner.setClassName(testClassName); + } else { + runner.setMethodName(testClassName, testMethodName); + } runner.setMaxtimeToOutputResponse(testRunParameters.getTestOutputTimeout()); if (testRunParameters.isCoverageEnabled()) {