Tests runorder

surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java

@@ -24,6 +24,7 @@
 import org.apache.maven.surefire.api.provider.CommandListener;
 import org.apache.maven.surefire.api.report.TestOutputReportEntry;
 import org.apache.maven.surefire.api.report.TestReportListener;
+import org.apache.maven.surefire.api.testset.ResolvedTest;
 import org.apache.maven.surefire.common.junit4.JUnit4RunListener;
 import org.apache.maven.surefire.common.junit4.JUnit4TestChecker;
 import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener;
@@ -50,11 +51,17 @@
 import org.junit.runner.manipulation.Filter;
 import org.junit.runner.notification.StoppedByUserException;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static java.lang.reflect.Modifier.isAbstract;
 import static java.lang.reflect.Modifier.isInterface;
+import static java.util.Collections.emptySet;
 import static org.apache.maven.surefire.api.report.RunMode.NORMAL_RUN;
 import static org.apache.maven.surefire.api.report.RunMode.RERUN_TEST_AFTER_FAILURE;
 import static org.apache.maven.surefire.common.junit4.JUnit4ProviderUtil.createMatchAnyDescriptionFilter;
@@ -78,6 +85,7 @@
 public class JUnit4Provider
     extends AbstractProvider
 {
+    private static final Pattern JUNIT_TEST_DESCRIPTION_PATTERN = Pattern.compile( "(.*)\\((.*)\\)" );
     private static final String UNDETERMINED_TESTS_DESCRIPTION = "cannot determine test in forked JVM with surefire";
 
     private final ClassMethodIndexer classMethodIndexer = new ClassMethodIndexer();
@@ -275,13 +283,14 @@ private void executeWithRerun( Class<?> clazz, Notifier notifier, RunModeSetter
         JUnitTestFailureListener failureListener = new JUnitTestFailureListener();
         notifier.addListener( failureListener );
         boolean hasMethodFilter = testResolver != null && testResolver.hasMethodPatterns();
+        Set<ResolvedTest> includedPatterns = testResolver != null ? testResolver.getIncludedPatterns() : emptySet();
 
         try
         {
             try
             {
                 notifier.asFailFast( isFailFast() );
-                execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null );
+                execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null, includedPatterns );
             }
             finally
             {
@@ -299,7 +308,7 @@ private void executeWithRerun( Class<?> clazz, Notifier notifier, RunModeSetter
                     Set<Description> failures = generateFailingTestDescriptions( failureListener.getAllFailures() );
                     failureListener.reset();
                     Filter failureDescriptionFilter = createMatchAnyDescriptionFilter( failures );
-                    execute( clazz, rerunNotifier, failureDescriptionFilter );
+                    execute( clazz, rerunNotifier, failureDescriptionFilter, includedPatterns );
                 }
             }
         }
@@ -361,12 +370,19 @@ private static boolean isJUnit4UpgradeCheck()
         return System.getProperty( "surefire.junit4.upgradecheck" ) != null;
     }
 
-    private static void execute( Class<?> testClass, Notifier notifier, Filter filter )
+    private static void execute( Class<?> testClass, Notifier notifier, Filter filter,
+                                 Set<ResolvedTest> includedPatterns )
     {
         final int classModifiers = testClass.getModifiers();
         if ( !isAbstract( classModifiers ) && !isInterface( classModifiers ) )
         {
             Request request = aClass( testClass );
+
+            if ( !includedPatterns.isEmpty() )
+            {
+                request = sortByTestOrder( includedPatterns, request );
+            }
+
             if ( filter != null )
             {
                 request = request.filterWith( filter );
@@ -379,6 +395,50 @@ private static void execute( Class<?> testClass, Notifier notifier, Filter filte
         }
     }
 
+    private static Request sortByTestOrder( Set<ResolvedTest> includedPatterns, Request request )
+    {
+        // This relies on the set being a LinkedHashSet (predictable iteration order)
+        List<ResolvedTest> testOrder = new ArrayList<>( includedPatterns );
+        Comparator<Description> descriptionComparator = ( d1, d2 ) ->
+        {
+            int i1 = indexOf( testOrder, d1 );
+            int i2 = indexOf( testOrder, d2 );
+            return i1 - i2;
+        };
+        return request.sortWith( descriptionComparator );
+    }
+
+    /**
+     * Finds the position of the ResolvedTest that matches the JUnit description
+     * @param orderedTests a test order
+     * @param description JUnit Description
+     * @return the position, or Integer.MAX_VALUE if not found
+     */
+    private static int indexOf( List<ResolvedTest> orderedTests, Description description )
+    {
+        if ( description.isSuite() )
+        {
+            return Integer.MAX_VALUE;
+        }
+        Matcher m = JUNIT_TEST_DESCRIPTION_PATTERN.matcher( description.toString() );
+        if ( !m.matches() )
+        {
+            return Integer.MAX_VALUE;
+        }
+        String methodName = m.group( 1 );
+        String className = m.group( 2 );
+        String classFileName = className.replace( ".", "/" ) + ".class";
+        for ( int i = 0; i < orderedTests.size(); i++ )
+        {
+            ResolvedTest resolvedTest = orderedTests.get( i );
+            if ( resolvedTest.matchAsInclusive( classFileName, methodName ) )
+            {
+                return i;
+            }
+        }
+        return Integer.MAX_VALUE;
+    }
+
     /**
      * JUnit error: test count includes one test-class as a suite which has filtered out all children.
      * Then the child test has a description "initializationError0(org.junit.runner.manipulation.Filter)"

surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderOrderingTest.java

@@ -0,0 +1,158 @@
+package org.apache.maven.surefire.junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.api.booter.BaseProviderFactory;
+import org.apache.maven.surefire.api.report.ReporterFactory;
+import org.apache.maven.surefire.api.report.TestOutputReportEntry;
+import org.apache.maven.surefire.api.report.TestReportListener;
+import org.apache.maven.surefire.api.suite.RunResult;
+import org.apache.maven.surefire.api.testset.RunOrderParameters;
+import org.apache.maven.surefire.api.testset.TestListResolver;
+import org.apache.maven.surefire.api.testset.TestRequest;
+import org.apache.maven.surefire.api.testset.TestSetFailedException;
+import org.apache.maven.surefire.api.util.TestsToRun;
+import org.junit.Ignore;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * @author Aslak Hellesøy
+ */
+public class JUnit4ProviderOrderingTest
+{
+    /**
+     */
+    public static class TestX
+    {
+        @Test
+        public void testA()
+        {
+        }
+
+        @Test
+        public void testB()
+        {
+        }
+
+        @Test
+        public void testC()
+        {
+        }
+    }
+
+    /**
+     */
+    public static class TestY
+    {
+        @Test
+        public void testD()
+        {
+        }
+
+        @Test
+        public void testE()
+        {
+        }
+
+        @Test
+        public void testF()
+        {
+        }
+
+    }
+
+    @Test
+    public void testShouldOrderWithinSingleTestClass() throws TestSetFailedException
+    {
+        assertTestOrder(
+            TestX.class.getName() + "#testC",
+            TestX.class.getName() + "#testA",
+            TestX.class.getName() + "#testB"
+        );
+    }
+
+    @Test
+    @Ignore( "this is currently not possible" )
+    public void testShouldOrderWithinInterleavedTestClasses() throws TestSetFailedException
+    {
+        assertTestOrder(
+            TestX.class.getName() + "#testC",
+            TestY.class.getName() + "#testE",
+            TestX.class.getName() + "#testA",
+            TestY.class.getName() + "#testD",
+            TestX.class.getName() + "#testB",
+            TestY.class.getName() + "#testF"
+        );
+    }
+
+    private void assertTestOrder( String... tests ) throws TestSetFailedException
+    {
+        List<String> testOrder = asList( tests );
+        BaseProviderFactory providerParameters = new BaseProviderFactory( true );
+        providerParameters.setProviderProperties( new HashMap<>() );
+        providerParameters.setClassLoaders( getClass().getClassLoader() );
+        TestListResolver requestedTests = new TestListResolver( String.join( ",", testOrder ) );
+        TestRequest testRequest = new TestRequest( null, null, requestedTests );
+        providerParameters.setTestRequest( testRequest );
+        providerParameters.setRunOrderParameters( RunOrderParameters.alphabetical() );
+
+        MockReporter testReportListener = new MockReporter();
+        providerParameters.setReporterFactory( new StubReporterFactory( testReportListener ) );
+        JUnit4Provider provider = new JUnit4Provider( providerParameters );
+        TestsToRun testsToRun = new TestsToRun( new HashSet<>( asList( TestX.class, TestY.class ) ) );
+        provider.invoke( testsToRun );
+
+        List<String> actualOrder = testReportListener.getReports().stream()
+            .map( r -> String.format( "%s#%s", r.getSourceName(), r.getName() ) )
+            .collect( Collectors.toList() );
+
+        assertThat( actualOrder, is( testOrder ) );
+    }
+
+    private static class StubReporterFactory implements ReporterFactory
+    {
+        private final TestReportListener<TestOutputReportEntry> testReportListener;
+
+        private StubReporterFactory( TestReportListener<TestOutputReportEntry> testReportListener )
+        {
+            this.testReportListener = testReportListener;
+        }
+
+        @Override
+        public TestReportListener<TestOutputReportEntry> createTestReportListener()
+        {
+            return testReportListener;
+        }
+
+        @Override
+        public RunResult close()
+        {
+            return null;
+        }
+    }
+}

surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4SuiteTest.java

@@ -22,6 +22,7 @@
 import junit.framework.JUnit4TestAdapter;
 import junit.framework.Test;
 import junit.framework.TestCase;
+import junit.framework.TestSuite;
 
 /**
  * Adapt the JUnit4 tests which use only annotations to the JUnit3 test suite.
@@ -30,6 +31,9 @@ public class JUnit4SuiteTest extends TestCase
 {
     public static Test suite()
     {
-        return new JUnit4TestAdapter( JUnit4ProviderTest.class );
+        TestSuite testSuite = new TestSuite();
+        testSuite.addTest( new JUnit4TestAdapter( JUnit4ProviderTest.class ) );
+        testSuite.addTest( new JUnit4TestAdapter( JUnit4ProviderOrderingTest.class ) );
+        return testSuite;
     }
 }