Implement fail-fast behavior for JUnit Platform provider

Author: marcphilipp

maven-surefire-plugin/src/site/apt/examples/skip-after-failure.apt.vm

@@ -41,8 +41,8 @@ Skipping Tests After the Nth Failure or Error
 
 Prerequisite
 
- Use ${project.artifactId} 2.19 or higher, JUnit 4.0 or higher, or 
- TestNG 5.10 or higher.
+ Use ${project.artifactId} 2.19 or higher, JUnit 4.0 or higher, TestNG 5.10 or
+ higher, or JUnit Platform 6.0 or higher.
 
  If version of TestNG is lower than 5.10 while the parameter 
  <<<skipAfterFailureCount>>> is set, the plugin fails with error:

surefire-providers/surefire-testng-utils/src/main/java/org/apache/maven/surefire/testng/utils/Stoppable.java renamed to surefire-api/src/main/java/org/apache/maven/surefire/api/report/Stoppable.java

@@ -16,17 +16,24 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.maven.surefire.testng.utils;
-
-import org.apache.maven.surefire.api.report.RunListener;
+package org.apache.maven.surefire.api.report;
 
 /**
  * Covers configuration parameter {@code skipAfterFailureCount}.
  *
  * @author <a href="mailto:[email protected]">Tibor Digana (tibor17)</a>
  * @since 2.19
  */
+@FunctionalInterface
 public interface Stoppable {
+
+    /**
+     * Null object for use when fail-fast behavior is disabled.
+     */
+    Stoppable NOOP = () -> {
+        // do nothing
+    };
+
     /**
      * Delegates this call to {@link RunListener#testExecutionSkippedByUser()}.
      */

surefire-its/src/test/java/org/apache/maven/surefire/its/AbstractFailFastIT.java

@@ -110,8 +110,14 @@ static Map<String, String> props(int forkCount, int skipAfterFailureCount, boole
     @Test
     public void test() throws Exception {
         String cls = useProcessPipes ? LEGACY_FORK_NODE : SUREFIRE_FORK_NODE;
-        prepare(description, profile, properties)
+        OutputValidator validator = prepare(description, profile, properties);
+        validator
                 .assertTestSuiteResults(total, errors, failures, skipped)
                 .assertThatLogLine(containsString("Found implementation of fork node factory: " + cls), equalTo(1));
+        performExtraChecks(validator);
+    }
+
+    protected void performExtraChecks(OutputValidator validator) throws Exception {
+        // may be overriden in subclasses
     }
 }

surefire-its/src/test/java/org/apache/maven/surefire/its/FailFastJUnitPlatformIT.java

@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.surefire.its;
+
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.junit.Test;
+import org.junit.runners.Parameterized.Parameters;
+
+import static java.lang.Integer.parseInt;
+import static org.apache.maven.surefire.its.fixture.HelperAssertions.assumeJavaVersion;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class FailFastJUnitPlatformIT extends AbstractFailFastIT {
+
+    @Parameters(name = "{0}")
+    public static Iterable<Object[]> data() {
+        //                    description    profile     forkCount,
+        //                    |              |           |  fail-fast-count,
+        //                    |              |           |  |  reuseForks
+        //                    |              |           |  |  |      total
+        //                    |              |           |  |  |      |  failures
+        //                    |              |           |  |  |      |  |  errors
+        //                    |              |           |  |  |      |  |  |  skipped
+        //                    |              |           |  |  |      |  |  |  |  pipes
+        return Arrays.asList( //             |           |  |  |      |  |  |  |  |
+                new Object[] {"oneFork-ff1", null, props(1, 1, true), 5, 0, 1, 4, true},
+                new Object[] {"oneFork-ff2", null, props(1, 2, true), 5, 0, 2, 3, true},
+                new Object[] {"twoForks-ff1", null, props(2, 1, true), 5, 0, 2, 3, true},
+                new Object[] {"twoForks-ff2", null, props(2, 2, true), 5, 0, 2, 2, true},
+                new Object[] {"oneFork-ff3", null, props(1, 3, true), 5, 0, 2, 0, true},
+                new Object[] {"twoForks-ff3", null, props(2, 3, true), 5, 0, 2, 0, true},
+                new Object[] {
+                    "unsupported-version", null, withUnsupportedJUnitVersion(props(2, 1, true)), 5, 0, 2, 0, true
+                });
+    }
+
+    private static Map<String, String> withUnsupportedJUnitVersion(Map<String, String> props) {
+        props.put("junit.version", "5.13.3");
+        return props;
+    }
+
+    @Override
+    protected String withProvider() {
+        return "junit-platform";
+    }
+
+    @Override
+    @Test
+    public void test() throws Exception {
+        if (!"unsupported-version".equals(description)) {
+            // JUnit 6.0.0 requires Java 17+
+            assumeJavaVersion(17);
+        }
+        super.test();
+    }
+
+    @Override
+    protected void performExtraChecks(OutputValidator validator) throws Exception {
+        if ("unsupported-version".equals(description)) {
+            int forkCount = parseInt(properties.get("forkCount"));
+            validator.assertThatLogLine(
+                    containsString("An attempt was made to cancel the current test run"), equalTo(forkCount));
+        }
+    }
+}

surefire-its/src/test/resources/fail-fast-junit-platform/pom.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>org.apache.maven.plugins.surefire</groupId>
+  <artifactId>fail-fast-junit-platform</artifactId>
+  <version>1.0</version>
+
+  <properties>
+    <maven.compiler.release>${java.specification.version}</maven.compiler.release>
+    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+    <junit.version>6.0.0-M2</junit.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>${surefire.version}</version>
+        <configuration>
+          <runOrder>alphabetical</runOrder>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.14.0</version>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

surefire-its/src/test/resources/fail-fast-junit-platform/src/test/java/pkg/ATest.java

@@ -0,0 +1,13 @@
+package pkg;
+
+import org.junit.jupiter.api.Test;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+class ATest {
+    @Test
+    void test() throws Exception {
+        MILLISECONDS.sleep(1_000);
+        throw new RuntimeException();
+    }
+}

surefire-its/src/test/resources/fail-fast-junit-platform/src/test/java/pkg/BTest.java

@@ -0,0 +1,13 @@
+package pkg;
+
+import org.junit.jupiter.api.Test;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+class BTest {
+    @Test
+    void test() throws Exception {
+        MILLISECONDS.sleep(2_000);
+        throw new RuntimeException();
+    }
+}

surefire-its/src/test/resources/fail-fast-junit-platform/src/test/java/pkg/CTest.java

@@ -0,0 +1,12 @@
+package pkg;
+
+import org.junit.jupiter.api.Test;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+class CTest {
+    @Test
+    void test() throws Exception {
+        MILLISECONDS.sleep(2_000);
+    }
+}

surefire-its/src/test/resources/fail-fast-junit-platform/src/test/java/pkg/DTest.java

@@ -0,0 +1,11 @@
+package pkg;
+
+import org.junit.jupiter.api.Test;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+class DTest {
+    @Test
+    void test() {
+    }
+}

surefire-its/src/test/resources/fail-fast-junit-platform/src/test/java/pkg/ETest.java

@@ -0,0 +1,11 @@
+package pkg;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+
+import org.junit.jupiter.api.Test;
+
+class ETest {
+    @Test
+    void test() {
+    }
+}