Consolidate summary line across multiple test results

apple · Nov 2, 2024 · 697c11e · 697c11e
1 parent 918942a
commit 697c11e
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 41 deletions.
diff --git a/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt b/pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt
@@ -20,6 +20,7 @@ import org.pkl.commons.cli.*
 import org.pkl.core.EvaluatorBuilder
 import org.pkl.core.ModuleSource.uri
 import org.pkl.core.Readers
+import org.pkl.core.TestResults
 import org.pkl.core.stdlib.test.report.JUnitReport
 import org.pkl.core.stdlib.test.report.SimpleReport
 import org.pkl.core.util.ErrorMessages
@@ -62,14 +63,17 @@ constructor(
       var failed = false
       var isExampleWrittenFailure = true
       val moduleNames = mutableSetOf<String>()
+      val reporter = SimpleReport(useColor)
+      val allTestResults = mutableListOf<TestResults>()
       for ((idx, moduleUri) in sources.withIndex()) {
         try {
           val results = evaluator.evaluateTest(uri(moduleUri), testOptions.overwrite)
+          allTestResults.add(results)
           if (!failed) {
             failed = results.failed()
             isExampleWrittenFailure = results.isExampleWrittenFailure.and(isExampleWrittenFailure)
           }
-          SimpleReport(useColor).report(results, consoleWriter)
+          reporter.report(results, consoleWriter)
           if (sources.size > 1 && idx != sources.size - 1) {
             consoleWriter.append('\n')
           }
@@ -102,6 +106,9 @@ constructor(
           failed = true
         }
       }
+      consoleWriter.append('\n')
+      reporter.summarize(allTestResults, consoleWriter)
+      consoleWriter.flush()
       if (failed) {
         val exitCode = if (isExampleWrittenFailure) 10 else 1
         throw CliTestException(ErrorMessages.create("testsFailed"), exitCode)

diff --git a/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt b/pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt
@@ -69,8 +69,8 @@ class CliTestRunnerTest {
       module test
         facts
           ✔ succeed
-
-      ✔ 100.0% tests pass [1 passed], 100.0% asserts pass [2 passed]
+      
+      100.0% tests pass [1 passed], 100.0% asserts pass [2 passed]
 
     """
           .trimIndent()
@@ -108,7 +108,7 @@ class CliTestRunnerTest {
             ✘ fail
                4 == 9 (/tempDir/test.pkl, line xx)
 
-        ✘ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]
+        0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]
 
         """
           .trimIndent()
@@ -150,7 +150,7 @@ class CliTestRunnerTest {
                  ^^^^^^^^^^^^^^
              at test#facts["fail"][#1] (/tempDir/test.pkl, line xx)
 
-      ✘ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+      0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
           .trimIndent()
@@ -192,7 +192,7 @@ class CliTestRunnerTest {
                  ^^^^^^^^^^^^^^
              at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
 
-      ✘ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+      0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
           .trimIndent()
@@ -248,7 +248,7 @@ class CliTestRunnerTest {
                  ^^^^^^^^^^^^^^
              at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
 
-      ✘ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+      0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
           .trimIndent()
@@ -447,7 +447,7 @@ class CliTestRunnerTest {
                (/tempDir/test.pkl, line xx)
                Output mismatch: Expected "nums" to contain 1 examples, but found 2
 
-        ✘ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+        0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
         """
           .trimIndent()
@@ -483,6 +483,7 @@ class CliTestRunnerTest {
           module test
             examples
               ✍️ nums
+
           1 examples written
           
           """

diff --git a/pkl-core/src/main/java/org/pkl/core/TestResults.java b/pkl-core/src/main/java/org/pkl/core/TestResults.java
@@ -92,7 +92,7 @@ public boolean failed() {
    * being written.
    */
   public boolean isExampleWrittenFailure() {
-    if (!failed() || !examples.failed()) return false;
+    if (!failed() || facts.failed() || !examples.failed()) return false;
     for (var testResult : examples.results) {
       if (!testResult.isExampleWritten) {
         return false;

diff --git a/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java b/pkl-core/src/main/java/org/pkl/core/stdlib/test/report/SimpleReport.java
@@ -17,11 +17,13 @@
 
 import java.io.IOException;
 import java.io.Writer;
+import java.util.List;
 import java.util.stream.Collectors;
 import org.pkl.core.TestResults;
 import org.pkl.core.TestResults.TestResult;
 import org.pkl.core.TestResults.TestSectionResults;
 import org.pkl.core.runtime.TextFormattingStringBuilder;
+import org.pkl.core.runtime.TextFormattingStringBuilder.AnsiCode;
 import org.pkl.core.util.ColorTheme;
 
 public final class SimpleReport implements TestReport {
@@ -45,37 +47,43 @@ public void report(TestResults results, Writer writer) throws IOException {
     if (results.error() != null) {
       var rendered = results.error().exception().getMessage();
       appendPadded(builder, rendered, "  ");
-      builder.append('\n');
+      builder.appendLine();
     } else {
       reportResults(results.facts(), builder);
       reportResults(results.examples(), builder);
     }
 
-    if (results.isExampleWrittenFailure()) {
-      builder.append(results.examples().totalFailures()).append(" examples written\n");
-      writer.append(builder.toString());
-      return;
-    }
-
-    builder.appendLine();
+    writer.append(builder.toString());
+  }
 
-    if (results.failed()) {
-      builder.append(ColorTheme.FAILING_TEST_MARK, failingMark);
+  public void summarize(List<TestResults> allTestResults, Writer writer) throws IOException {
+    var totalTests = 0;
+    var totalFailedTests = 0;
+    var totalAsserts = 0;
+    var totalFailedAsserts = 0;
+    var isFailed = false;
+    var isExampleWrittenFailure = true;
+    for (var testResults : allTestResults) {
+      if (!isFailed) {
+        isFailed = testResults.failed();
+      }
+      if (testResults.failed()) {
+        isExampleWrittenFailure = testResults.isExampleWrittenFailure() & isExampleWrittenFailure;
+      }
+      totalTests += testResults.totalTests();
+      totalFailedTests += testResults.totalFailures();
+      totalAsserts += testResults.totalAsserts();
+      totalFailedAsserts += testResults.totalAssertsFailed();
+    }
+    var builder = new TextFormattingStringBuilder(useColor);
+    if (isFailed && isExampleWrittenFailure) {
+      builder.append(totalFailedTests).append(" examples written");
     } else {
-      builder.append(ColorTheme.PASSING_TEST_MARK, passingMark);
+      makeStatsLine(builder, "tests", totalTests, totalFailedTests, isFailed);
+      builder.append(", ");
+      makeStatsLine(builder, "asserts", totalAsserts, totalFailedAsserts, isFailed);
     }
-
-    var totalStatsLine =
-        makeStatsLine("tests", results.totalTests(), results.totalFailures(), results.failed());
-    builder.append(totalStatsLine);
-
-    var totalAssertsStatsLine =
-        makeStatsLine(
-            "asserts", results.totalAsserts(), results.totalAssertsFailed(), results.failed());
-    builder.append(", ").append(totalAssertsStatsLine);
-
-    builder.append("\n");
-
+    builder.appendLine();
     writer.append(builder.toString());
   }
 
@@ -124,18 +132,22 @@ private static void appendPadded(
         });
   }
 
-  private String makeStatsLine(String kind, int total, int failed, boolean isFailed) {
+  private void makeStatsLine(
+      TextFormattingStringBuilder sb, String kind, int total, int failed, boolean isFailed) {
     var passed = total - failed;
     var passRate = total > 0 ? 100.0 * passed / total : 0.0;
 
-    String line = String.format("%.1f%% %s pass", passRate, kind);
+    var color = isFailed ? AnsiCode.RED : AnsiCode.GREEN;
+    sb.append(
+        color,
+        () -> {
+          sb.append(String.format("%.1f%%", passRate)).append(" ").append(kind).append(" pass");
+        });
 
     if (isFailed) {
-      line += String.format(" [%d/%d failed]", failed, total);
+      sb.append(" [").append(failed).append('/').append(total).append(" failed]");
     } else {
-      line += String.format(" [%d passed]", passed);
+      sb.append(" [").append(passed).append(" passed]");
     }
-
-    return line;
   }
 }
diff --git a/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt b/pkl-gradle/src/test/kotlin/org/pkl/gradle/TestsTest.kt
@@ -90,7 +90,7 @@ class TestsTest : AbstractTest() {
                  ^^^^^^^^^^^^^^^^^^
              at test#facts["error"][#1] (file:///file, line x)
 
-      ✘ 50.0% tests pass [1/2 failed], 66.7% asserts pass [1/3 failed]
+      50.0% tests pass [1/2 failed], 66.7% asserts pass [1/3 failed]
       """
           .trimIndent()
       )
@@ -139,7 +139,7 @@ class TestsTest : AbstractTest() {
                    age = 35
                  }
 
-        ✘ 60.0% tests pass [2/5 failed], 66.7% asserts pass [3/9 failed]
+        60.0% tests pass [2/5 failed], 66.7% asserts pass [3/9 failed]
         """
           .trimIndent()
       )
@@ -212,7 +212,7 @@ class TestsTest : AbstractTest() {
                    age = 35
                  }
 
-        ✘ 50.0% tests pass [2/4 failed], 66.7% asserts pass [2/6 failed]
+        50.0% tests pass [2/4 failed], 66.7% asserts pass [2/6 failed]
 
         """
           .trimIndent()