Tests runorder

surefire-providers/surefire-junit4/src/main/java/org/apache/maven/surefire/junit4/JUnit4Provider.java

@@ -24,6 +24,7 @@
 import org.apache.maven.surefire.api.provider.CommandListener;
 import org.apache.maven.surefire.api.report.TestOutputReportEntry;
 import org.apache.maven.surefire.api.report.TestReportListener;
+import org.apache.maven.surefire.api.testset.ResolvedTest;
 import org.apache.maven.surefire.common.junit4.JUnit4RunListener;
 import org.apache.maven.surefire.common.junit4.JUnit4TestChecker;
 import org.apache.maven.surefire.common.junit4.JUnitTestFailureListener;
@@ -50,8 +51,13 @@
 import org.junit.runner.manipulation.Filter;
 import org.junit.runner.notification.StoppedByUserException;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import static java.lang.reflect.Modifier.isAbstract;
 import static java.lang.reflect.Modifier.isInterface;
@@ -270,18 +276,57 @@ private void executeTestSet( Class<?> clazz, RunListener reporter, Notifier noti
         }
     }
 
+    private static int indexOf( List<ResolvedTest> tests, Description description )
+    {
+        if ( description.isSuite() )
+        {
+            return Integer.MAX_VALUE;
+        }
+        Pattern p = Pattern.compile( "(.*)\\((.*)\\)" );
+        Matcher m = p.matcher( description.toString() );
+        if ( !m.matches() )
+        {
+            return Integer.MAX_VALUE;
+        }
+        String methodName = m.group( 1 );
+        String className = m.group( 2 );
+        String classFileName = className.replace( ".", "/" ) + ".class";
+        for ( int i = 0; i < tests.size(); i++ )
+        {
+            ResolvedTest resolvedTest = tests.get( i );
+            if ( resolvedTest.matchAsInclusive( classFileName, methodName ) )
+            {
+                return i;
+            }
+        }
+        return Integer.MAX_VALUE;
+    }
+
     private void executeWithRerun( Class<?> clazz, Notifier notifier, RunModeSetter runMode )
     {
         JUnitTestFailureListener failureListener = new JUnitTestFailureListener();
         notifier.addListener( failureListener );
         boolean hasMethodFilter = testResolver != null && testResolver.hasMethodPatterns();
 
+        // This relies on the set being a LinkedHashSet (predictable iteration order)
+        List<ResolvedTest> includedPatterns = new ArrayList<>( testResolver.getIncludedPatterns() );
+        Comparator<Description> descriptionComparator = new Comparator<Description>()
+        {
+            @Override
+            public int compare( Description d1, Description d2 )
+            {
+                int i1 = indexOf( includedPatterns, d1 );
+                int i2 = indexOf( includedPatterns, d2 );
+                return i1 - i2;
+            }
+        };
+
         try
         {
             try
             {
                 notifier.asFailFast( isFailFast() );
-                execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null );
+                execute( clazz, notifier, hasMethodFilter ? createMethodFilter() : null, descriptionComparator );
             }
             finally
             {
@@ -299,7 +344,7 @@ private void executeWithRerun( Class<?> clazz, Notifier notifier, RunModeSetter
                     Set<Description> failures = generateFailingTestDescriptions( failureListener.getAllFailures() );
                     failureListener.reset();
                     Filter failureDescriptionFilter = createMatchAnyDescriptionFilter( failures );
-                    execute( clazz, rerunNotifier, failureDescriptionFilter );
+                    execute( clazz, rerunNotifier, failureDescriptionFilter, descriptionComparator );
                 }
             }
         }
@@ -361,12 +406,14 @@ private static boolean isJUnit4UpgradeCheck()
         return System.getProperty( "surefire.junit4.upgradecheck" ) != null;
     }
 
-    private static void execute( Class<?> testClass, Notifier notifier, Filter filter )
+    private static void execute( Class<?> testClass, Notifier notifier, Filter filter,
+                                 Comparator<Description> descriptionComparator )
     {
         final int classModifiers = testClass.getModifiers();
         if ( !isAbstract( classModifiers ) && !isInterface( classModifiers ) )
         {
             Request request = aClass( testClass );
+            request = request.sortWith( descriptionComparator );
             if ( filter != null )
             {
                 request = request.filterWith( filter );

surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/JUnit4ProviderOrderingTest.java

@@ -0,0 +1,143 @@
+package org.apache.maven.surefire.junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.api.booter.BaseProviderFactory;
+import org.apache.maven.surefire.api.report.ReporterFactory;
+import org.apache.maven.surefire.api.report.TestOutputReportEntry;
+import org.apache.maven.surefire.api.report.TestReportListener;
+import org.apache.maven.surefire.api.suite.RunResult;
+import org.apache.maven.surefire.api.testset.RunOrderParameters;
+import org.apache.maven.surefire.api.testset.TestListResolver;
+import org.apache.maven.surefire.api.testset.TestRequest;
+import org.apache.maven.surefire.api.testset.TestSetFailedException;
+import org.apache.maven.surefire.api.util.TestsToRun;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Arrays.asList;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * @author Aslak Hellesøy
+ */
+public class JUnit4ProviderOrderingTest
+{
+    /**
+     */
+    public static class TestX
+    {
+        @Test
+        public void testA()
+        {
+        }
+
+        @Test
+        public void testB()
+        {
+        }
+
+        @Test
+        public void testC()
+        {
+        }
+    }
+
+    /**
+     */
+    public static class TestY
+    {
+        @Test
+        public void testD()
+        {
+        }
+
+        @Test
+        public void testE()
+        {
+        }
+
+        @Test
+        public void testF()
+        {
+        }
+
+    }
+
+    @Test
+    public void testShouldOrderWithinSingleTestClass() throws TestSetFailedException
+    {
+        assertTestOrder(
+            TestX.class.getName() + "#testC",
+            TestX.class.getName() + "#testA",
+            TestX.class.getName() + "#testB"
+        );
+    }
+
+    private void assertTestOrder( String... tests ) throws TestSetFailedException
+    {
+        List<String> testOrder = asList( tests );
+        BaseProviderFactory providerParameters = new BaseProviderFactory( true );
+        providerParameters.setProviderProperties( new HashMap<>() );
+        providerParameters.setClassLoaders( getClass().getClassLoader() );
+        TestListResolver requestedTests = new TestListResolver( String.join( ",", testOrder ) );
+        TestRequest testRequest = new TestRequest( null, null, requestedTests );
+        providerParameters.setTestRequest( testRequest );
+        providerParameters.setRunOrderParameters( RunOrderParameters.alphabetical() );
+
+        MockReporter testReportListener = new MockReporter();
+        providerParameters.setReporterFactory( new StubReporterFactory( testReportListener ) );
+        JUnit4Provider provider = new JUnit4Provider( providerParameters );
+        TestsToRun testsToRun = new TestsToRun( new HashSet<>( asList( TestX.class, TestY.class ) ) );
+        provider.invoke( testsToRun );
+
+        List<String> actualOrder = testReportListener.getReports().stream()
+            .map( r -> String.format( "%s#%s", r.getSourceName(), r.getName() ) )
+            .collect( Collectors.toList() );
+
+        assertThat( actualOrder, is( testOrder ) );
+    }
+
+    private static class StubReporterFactory implements ReporterFactory
+    {
+        private final TestReportListener<TestOutputReportEntry> testReportListener;
+
+        private StubReporterFactory( TestReportListener<TestOutputReportEntry> testReportListener )
+        {
+            this.testReportListener = testReportListener;
+        }
+
+        @Override
+        public TestReportListener<TestOutputReportEntry> createTestReportListener()
+        {
+            return testReportListener;
+        }
+
+        @Override
+        public RunResult close()
+        {
+            return null;
+        }
+    }
+}

surefire-providers/surefire-junit4/src/test/java/org/apache/maven/surefire/junit4/MockReporter.java

@@ -0,0 +1,147 @@
+package org.apache.maven.surefire.junit4;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import org.apache.maven.surefire.api.report.ReportEntry;
+import org.apache.maven.surefire.api.report.TestOutputReportEntry;
+import org.apache.maven.surefire.api.report.TestReportListener;
+import org.apache.maven.surefire.api.report.TestSetReportEntry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Internal tests use only.
+ */
+final class MockReporter
+        implements TestReportListener<TestOutputReportEntry>
+{
+    private final List<ReportEntry> reports = new ArrayList<>();
+
+    @Override
+    public void testSetStarting( TestSetReportEntry report )
+    {
+    }
+
+    @Override
+    public void testSetCompleted( TestSetReportEntry report )
+    {
+    }
+
+    @Override
+    public void testStarting( ReportEntry report )
+    {
+        reports.add( report );
+    }
+
+    @Override
+    public void testSucceeded( ReportEntry report )
+    {
+    }
+
+    @Override
+    public void testSkipped( ReportEntry report )
+    {
+    }
+
+    @Override
+    public void testExecutionSkippedByUser()
+    {
+    }
+
+    @Override
+    public void testError( ReportEntry report )
+    {
+    }
+
+    @Override
+    public void testFailed( ReportEntry report )
+    {
+    }
+
+    @Override
+    public void testAssumptionFailure( ReportEntry report )
+    {
+    }
+
+    @Override
+    public void writeTestOutput( TestOutputReportEntry reportEntry )
+    {
+    }
+
+    @Override
+    public boolean isDebugEnabled()
+    {
+        return false;
+    }
+
+    @Override
+    public void debug( String message )
+    {
+    }
+
+    @Override
+    public boolean isInfoEnabled()
+    {
+        return false;
+    }
+
+    @Override
+    public void info( String message )
+    {
+    }
+
+    @Override
+    public boolean isWarnEnabled()
+    {
+        return false;
+    }
+
+    @Override
+    public void warning( String message )
+    {
+    }
+
+    @Override
+    public boolean isErrorEnabled()
+    {
+        return false;
+    }
+
+    @Override
+    public void error( String message )
+    {
+    }
+
+    @Override
+    public void error( String message, Throwable t )
+    {
+    }
+
+    @Override
+    public void error( Throwable t )
+    {
+    }
+
+    public List<ReportEntry> getReports()
+    {
+        return reports;
+    }
+}