apache · Frawless · Oct 8, 2025
diff --git a/...-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java b/...-common/src/main/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactory.java
@@ -275,24 +275,32 @@ private void mergeTestHistoryResult() {
             List<TestMethodStats> testMethodStats = entry.getValue();
             String testClassMethodName = entry.getKey();
 
-            // Handle @BeforeAll failures (null method names)
-            if (testClassMethodName == null) {
-                for (TestMethodStats methodStats : testMethodStats) {
-                    String className = extractClassNameFromStackTrace(methodStats);
+            // Handle @BeforeAll failures (null, empty, or ends with ".null" method names)
+            // But only if they actually failed (ERROR or FAILURE), not if they were skipped
+            if ((testClassMethodName == null || testClassMethodName.isEmpty() || testClassMethodName.endsWith(".null"))
+                    && (testClassMethodName == null || !testClassMethodName.contains("$"))) {
+
+                // Check if this is actually a failure/error (not skipped or success)
+                boolean isActualFailure = testMethodStats.stream()
+                        .anyMatch(stat -> stat.getResultType() == ReportEntryType.ERROR
+                                || stat.getResultType() == ReportEntryType.FAILURE);
+
+                if (isActualFailure) {
+                    // Extract class name from the test class method name
+                    String className = extractClassNameFromMethodName(testClassMethodName);
                     if (className != null) {
                         if (beforeAllFailures.containsKey(className)) {
                             List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
-                            previousMethodStats.add(methodStats);
+                            previousMethodStats.addAll(testMethodStats);
                             beforeAllFailures.put(className, previousMethodStats);
                         } else {
-                            List<TestMethodStats> initMethodStats = new ArrayList<>();
-                            initMethodStats.add(methodStats);
-                            beforeAllFailures.put(className, initMethodStats);
+                            beforeAllFailures.put(className, new ArrayList<>(testMethodStats));
                         }
                     }
+                    // Skip normal processing of @BeforeAll failures because it needs special care
+                    continue;
                 }
-                // Skip normal processing of @BeforeAll failures because it needs special care
-                continue;
+                // If it's skipped or success with null method name, fall through to normal processing
             }
 
             completedCount++;
@@ -313,15 +321,6 @@ private void mergeTestHistoryResult() {
                     }
                     completedCount += successCount - 1;
                     successTests.put(testClassMethodName, testMethodStats);
-
-                    // If current test belong to class that failed during beforeAll store that info to proper log info
-                    String className = extractClassNameFromMethodName(testClassMethodName);
-                    if (beforeAllFailures.containsKey(className)) {
-                        List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
-                        previousMethodStats.addAll(testMethodStats);
-                        beforeAllFailures.put(className, previousMethodStats);
-                    }
-
                     break;
                 case SKIPPED:
                     skipped++;
@@ -340,14 +339,30 @@ private void mergeTestHistoryResult() {
             }
         }
 
+        // Loop over all success tests and find those that are passed flakes for beforeAll failures
+        for (Map.Entry<String, List<TestMethodStats>> entry : successTests.entrySet()) {
+            List<TestMethodStats> testMethodStats = entry.getValue();
+            String testClassMethodName = entry.getKey();
+            // If current test belong to class that failed during beforeAll store that info to proper log info
+            String className = extractClassNameFromMethodName(testClassMethodName);
+            if (beforeAllFailures.containsKey(className)) {
+                List<TestMethodStats> previousMethodStats = beforeAllFailures.get(className);
+                previousMethodStats.addAll(testMethodStats);
+                beforeAllFailures.put(className, previousMethodStats);
+            }
+        }
+
         // Process @BeforeAll failures after we know which classes have successful tests
         for (Map.Entry<String, List<TestMethodStats>> entry : beforeAllFailures.entrySet()) {
             String className = entry.getKey();
             List<TestMethodStats> testMethodStats = entry.getValue();
             String classNameKey = className + ".<beforeAll>";
 
             if (reportConfiguration.getRerunFailingTestsCount() > 0
-                    && testMethodStats.stream().anyMatch(methodStats -> methodStats.getTestClassMethodName() != null)) {
+                    && testMethodStats.stream()
+                            .anyMatch(methodStats -> methodStats.getTestClassMethodName() != null
+                                    && !methodStats.getTestClassMethodName().isEmpty()
+                                    && methodStats.getResultType().equals(ReportEntryType.SUCCESS))) {
                 flakyTests.put(classNameKey, testMethodStats);
             } else {
                 errorTests.put(classNameKey, testMethodStats);

diff --git a/...re-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java b/...re-common/src/main/java/org/apache/maven/plugin/surefire/report/StatelessXmlReporter.java
@@ -161,7 +161,7 @@ public void testSetCompleted(WrappedReportEntry testSetReportEntry, TestSetStats
                 OutputStreamWriter fw = getWriter(outputStream)) {
             XMLWriter ppw = new PrettyPrintXMLWriter(new PrintWriter(fw), XML_INDENT, XML_NL, UTF_8.name(), null);
 
-            createTestSuiteElement(ppw, testSetReportEntry, testSetStats); // TestSuite
+            createTestSuiteElement(ppw, testSetReportEntry, classMethodStatistics); // TestSuite
 
             if (enablePropertiesElement) {
                 showProperties(ppw, testSetReportEntry.getSystemProperties());
@@ -186,9 +186,9 @@ public void testSetCompleted(WrappedReportEntry testSetReportEntry, TestSetStats
             }
 
             for (Entry<String, Map<String, List<WrappedReportEntry>>> statistics : classMethodStatistics.entrySet()) {
-                for (Entry<String, List<WrappedReportEntry>> thisMethodRuns :
-                        statistics.getValue().entrySet()) {
-                    serializeTestClass(outputStream, fw, ppw, thisMethodRuns.getValue());
+                Map<String, List<WrappedReportEntry>> methodStatistics = statistics.getValue();
+                for (Entry<String, List<WrappedReportEntry>> thisMethodRuns : methodStatistics.entrySet()) {
+                    serializeTestClass(outputStream, fw, ppw, thisMethodRuns.getValue(), methodStatistics);
                 }
             }
 
@@ -224,10 +224,14 @@ private Deque<WrappedReportEntry> aggregateCacheFromMultipleReruns(
     }
 
     private void serializeTestClass(
-            OutputStream outputStream, OutputStreamWriter fw, XMLWriter ppw, List<WrappedReportEntry> methodEntries)
+            OutputStream outputStream,
+            OutputStreamWriter fw,
+            XMLWriter ppw,
+            List<WrappedReportEntry> methodEntries,
+            Map<String, List<WrappedReportEntry>> methodStatistics)
             throws IOException {
         if (rerunFailingTestsCount > 0) {
-            serializeTestClassWithRerun(outputStream, fw, ppw, methodEntries);
+            serializeTestClassWithRerun(outputStream, fw, ppw, methodEntries, methodStatistics);
         } else {
             // rerunFailingTestsCount is smaller than 1, but for some reasons a test could be run
             // for more than once
@@ -258,10 +262,18 @@ private void serializeTestClassWithoutRerun(
     }
 
     private void serializeTestClassWithRerun(
-            OutputStream outputStream, OutputStreamWriter fw, XMLWriter ppw, List<WrappedReportEntry> methodEntries)
+            OutputStream outputStream,
+            OutputStreamWriter fw,
+            XMLWriter ppw,
+            List<WrappedReportEntry> methodEntries,
+            Map<String, List<WrappedReportEntry>> methodStatistics)
             throws IOException {
         WrappedReportEntry firstMethodEntry = methodEntries.get(0);
-        switch (getTestResultType(methodEntries)) {
+
+        TestResultType resultType =
+                getTestResultTypeWithBeforeAllHandling(firstMethodEntry.getName(), methodEntries, methodStatistics);
+
+        switch (resultType) {
             case SUCCESS:
                 for (WrappedReportEntry methodEntry : methodEntries) {
                     if (methodEntry.getReportEntryType() == SUCCESS) {
@@ -370,6 +382,40 @@ private TestResultType getTestResultType(List<WrappedReportEntry> methodEntryLis
         return DefaultReporterFactory.getTestResultType(testResultTypeList, rerunFailingTestsCount);
     }
 
+    /**
+     * Determines the final result type for a test method, applying special handling for @BeforeAll failures.
+     * If a @BeforeAll fails but any actual test methods succeed, it's classified as a FLAKE.
+     *
+     * @param methodName the name of the test method (null or "null" for @BeforeAll)
+     * @param methodRuns the list of runs for this method
+     * @param methodStatistics all method statistics for the test class
+     * @return the final TestResultType
+     */
+    private TestResultType getTestResultTypeWithBeforeAllHandling(
+            String methodName,
+            List<WrappedReportEntry> methodRuns,
+            Map<String, List<WrappedReportEntry>> methodStatistics) {
+        TestResultType resultType = getTestResultType(methodRuns);
+
+        // Special handling for @BeforeAll failures (null method name or method name is "null")
+        // If @BeforeAll failed but any actual test methods succeeded, treat it as a flake
+        if ((methodName == null || methodName.equals("null"))
+                && (resultType == TestResultType.ERROR || resultType == TestResultType.FAILURE)) {
+            // Check if any actual test methods (non-null and not "null" names) succeeded
+            boolean hasSuccessfulTestMethods = methodStatistics.entrySet().stream()
+                    .filter(entry ->
+                            entry.getKey() != null && !entry.getKey().equals("null")) // Only actual test methods
+                    .anyMatch(entry -> entry.getValue().stream()
+                            .anyMatch(reportEntry -> reportEntry.getReportEntryType() == SUCCESS));
+
+            if (hasSuccessfulTestMethods) {
+                resultType = TestResultType.FLAKE;
+            }
+        }
+
+        return resultType;
+    }
+
     private Deque<WrappedReportEntry> getAddMethodRunHistoryMap(String testClassName) {
         Deque<WrappedReportEntry> methodRunHistory = testClassMethodRunHistoryMap.get(testClassName);
         if (methodRunHistory == null) {
@@ -420,7 +466,10 @@ private void startTestElement(XMLWriter ppw, WrappedReportEntry report) throws I
         }
     }
 
-    private void createTestSuiteElement(XMLWriter ppw, WrappedReportEntry report, TestSetStats testSetStats)
+    private void createTestSuiteElement(
+            XMLWriter ppw,
+            WrappedReportEntry report,
+            Map<String, Map<String, List<WrappedReportEntry>>> classMethodStatistics)
             throws IOException {
         ppw.startElement("testsuite");
 
@@ -441,13 +490,46 @@ private void createTestSuiteElement(XMLWriter ppw, WrappedReportEntry report, Te
             ppw.addAttribute("time", String.valueOf(report.getElapsed() / ONE_SECOND));
         }
 
-        ppw.addAttribute("tests", String.valueOf(testSetStats.getCompletedCount()));
-
-        ppw.addAttribute("errors", String.valueOf(testSetStats.getErrors()));
-
-        ppw.addAttribute("skipped", String.valueOf(testSetStats.getSkipped()));
+        // Count actual unique test methods and their final results from classMethodStatistics (accumulated across
+        // reruns)
+        int actualTestCount = 0;
+        int errors = 0;
+        int failures = 0;
+        int skipped = 0;
+        int flakes = 0;
+
+        for (Map<String, List<WrappedReportEntry>> methodStats : classMethodStatistics.values()) {
+            actualTestCount += methodStats.size();
+            for (Map.Entry<String, List<WrappedReportEntry>> methodEntry : methodStats.entrySet()) {
+                String methodName = methodEntry.getKey();
+                List<WrappedReportEntry> methodRuns = methodEntry.getValue();
+                TestResultType resultType = getTestResultTypeWithBeforeAllHandling(methodName, methodRuns, methodStats);
+
+                switch (resultType) {
+                    case ERROR:
+                        errors++;
+                        break;
+                    case FAILURE:
+                        failures++;
+                        break;
+                    case SKIPPED:
+                        skipped++;
+                        break;
+                    case FLAKE:
+                        flakes++;
+                        break;
+                    case SUCCESS:
+                    default:
+                        break;
+                }
+            }
+        }
 
-        ppw.addAttribute("failures", String.valueOf(testSetStats.getFailures()));
+        ppw.addAttribute("tests", String.valueOf(actualTestCount));
+        ppw.addAttribute("errors", String.valueOf(errors));
+        ppw.addAttribute("skipped", String.valueOf(skipped));
+        ppw.addAttribute("failures", String.valueOf(failures));
+        ppw.addAttribute("flakes", String.valueOf(flakes));
     }
 
     private static void getTestProblems(

diff --git a/...mon/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java b/...mon/src/test/java/org/apache/maven/plugin/surefire/report/DefaultReporterFactoryTest.java
@@ -114,7 +114,7 @@ public void testMergeTestHistoryResult() throws Exception {
         firstRunStats.add(new TestMethodStats(TEST_FIVE, ReportEntryType.SUCCESS, null));
         // @BeforeAll failure for a test class that will eventually succeed
         firstRunStats.add(new TestMethodStats(
-                null,
+                TEST_BEFORE_ALL_FLAKE + ".null",
                 ReportEntryType.ERROR,
                 new DummyStackTraceWriter(TEST_BEFORE_ALL_FLAKE + ".null " + TEST_ERROR_SUFFIX)));
 
@@ -129,7 +129,7 @@ public void testMergeTestHistoryResult() throws Exception {
         secondRunStats.add(new TestMethodStats(TEST_BEFORE_ALL_FLAKE + ".testSucceed", ReportEntryType.SUCCESS, null));
         // @BeforeAll failure for a different class that will stay as error
         secondRunStats.add(new TestMethodStats(
-                null,
+                TEST_BEFORE_ALL_ERROR + ".null",
                 ReportEntryType.ERROR,
                 new DummyStackTraceWriter(TEST_BEFORE_ALL_ERROR + ".null " + TEST_ERROR_SUFFIX)));
 
@@ -139,7 +139,9 @@ public void testMergeTestHistoryResult() throws Exception {
         thirdRunStats.add(new TestMethodStats(TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter(ERROR)));
         // Another @BeforeAll failure for the always-failing class
         thirdRunStats.add(new TestMethodStats(
-                null, ReportEntryType.ERROR, new DummyStackTraceWriter(TEST_BEFORE_ALL_ERROR + ".null")));
+                TEST_BEFORE_ALL_ERROR + ".null",
+                ReportEntryType.ERROR,
+                new DummyStackTraceWriter(TEST_BEFORE_ALL_ERROR + ".null")));
 
         TestSetRunListener firstRunListener = mock(TestSetRunListener.class);
         TestSetRunListener secondRunListener = mock(TestSetRunListener.class);

diff --git a/...its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformFailingBeforeAllRerunIT.java b/...its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformFailingBeforeAllRerunIT.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.maven.surefire.its;
+
+import org.apache.maven.surefire.its.fixture.OutputValidator;
+import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
+import org.junit.Test;
+
+/**
+ * Integration tests for JUnit Platform @BeforeAll failures with rerun functionality.
+ * Tests various scenarios where @BeforeAll lifecycle methods fail and are rerun.
+ */
+public class JUnitPlatformFailingBeforeAllRerunIT extends SurefireJUnit4IntegrationTestCase {
+    private static final String VERSION = "5.9.1";
+
+    private static final String TEST_PROJECT_BASE = "junit-platform-rerun-failing-before-all";
+
+    @Test
+    public void testBeforeAllFailures() {
+        // Test that @BeforeAll failures are properly handled when they succeed on rerun
+        OutputValidator outputValidator = unpack(TEST_PROJECT_BASE)
+                .setJUnitVersion(VERSION)
+                .maven()
+                .debugLogging()
+                .addGoal("-Dsurefire.rerunFailingTestsCount=3")
+                .withFailure()
+                .executeTest()
+                .assertTestSuiteResults(7, 1, 0, 0, 4);
+
+        // Verify the @BeforeAll is reported as a flake with proper formatting
+        outputValidator.verifyTextInLog("junitplatform.FlakyFirstTimeTest.<beforeAll>");
+        outputValidator.verifyTextInLog("Run 1: FlakyFirstTimeTest.setup:53 IllegalArgument");
+        outputValidator.verifyTextInLog("Run 2: PASS");
+
+        // Verify XML report doesn't contain error testcase with empty name
+        outputValidator
+                .getSurefireReportsXmlFile("TEST-junitplatform.FlakyFirstTimeTest.xml")
+                .assertContainsText("tests=\"4\" errors=\"0\"")
+                .assertContainsText("name=\"testFailingTestOne\"")
+                .assertContainsText("name=\"testErrorTestOne\"")
+                .assertContainsText("name=\"testPassingTest\"");
+
+        // Verify @BeforeAll is reported as error
+        outputValidator.verifyTextInLog("Errors:");
+        outputValidator.verifyTextInLog("junitplatform.AlwaysFailingTest.<beforeAll>");
+        outputValidator.verifyTextInLog("Run 3: AlwaysFailingTest.setup:15 IllegalArgument BeforeAll always fails");
+    }
+}