Use ANSI colors for test results; more polish

pkl-cli/src/main/kotlin/org/pkl/cli/CliTestRunner.kt

@@ -20,6 +20,7 @@ import org.pkl.commons.cli.*
 import org.pkl.core.Closeables
 import org.pkl.core.EvaluatorBuilder
 import org.pkl.core.ModuleSource.uri
+import org.pkl.core.TestResults
 import org.pkl.core.stdlib.test.report.JUnitReport
 import org.pkl.core.stdlib.test.report.SimpleReport
 import org.pkl.core.util.ErrorMessages
@@ -62,14 +63,17 @@ constructor(
       var failed = false
       var isExampleWrittenFailure = true
       val moduleNames = mutableSetOf<String>()
+      val reporter = SimpleReport(useColor)
+      val allTestResults = mutableListOf<TestResults>()
       for ((idx, moduleUri) in sources.withIndex()) {
         try {
           val results = evaluator.evaluateTest(uri(moduleUri), testOptions.overwrite)
+          allTestResults.add(results)
           if (!failed) {
             failed = results.failed()
             isExampleWrittenFailure = results.isExampleWrittenFailure.and(isExampleWrittenFailure)
           }
-          SimpleReport().report(results, consoleWriter)
+          reporter.report(results, consoleWriter)
           if (sources.size > 1 && idx != sources.size - 1) {
             consoleWriter.append('\n')
           }
@@ -102,6 +106,9 @@ constructor(
           failed = true
         }
       }
+      consoleWriter.append('\n')
+      reporter.summarize(allTestResults, consoleWriter)
+      consoleWriter.flush()
       if (failed) {
         val exitCode = if (isExampleWrittenFailure) 10 else 1
         throw CliTestException(ErrorMessages.create("testsFailed"), exitCode)

pkl-cli/src/test/kotlin/org/pkl/cli/CliTestRunnerTest.kt

@@ -37,7 +37,6 @@ import org.pkl.commons.writeString
 import org.pkl.core.Release
 
 class CliTestRunnerTest {
-
   @Test
   fun `CliTestRunner succeed test`(@TempDir tempDir: Path) {
     val code =
@@ -65,8 +64,9 @@ class CliTestRunnerTest {
         """
       module test
         facts
-          ✅ succeed
-      ✅ 100.0% tests pass [1 passed], 100.0% asserts pass [2 passed]
+          ✔ succeed
+
+      100.0% tests pass [1 passed], 100.0% asserts pass [2 passed]
 
     """
           .trimIndent()
@@ -101,9 +101,10 @@ class CliTestRunnerTest {
         """
         module test
           facts
-            ❌ fail
+            ✘ fail
                4 == 9 (/tempDir/test.pkl, line xx)
-        ❌ 0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]
+
+        0.0% tests pass [1/1 failed], 50.0% asserts pass [1/2 failed]
 
         """
           .trimIndent()
@@ -137,14 +138,15 @@ class CliTestRunnerTest {
         """
       module test
         facts
-          ❌ fail
+          ✘ fail
              –– Pkl Error ––
              uh oh
 
              5 | throw("uh oh")
                  ^^^^^^^^^^^^^^
              at test#facts["fail"][#1] (/tempDir/test.pkl, line xx)
-      ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+
+      0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
           .trimIndent()
@@ -178,14 +180,15 @@ class CliTestRunnerTest {
         """
       module test
         examples
-          ❌ fail
+          ✘ fail
              –– Pkl Error ––
              uh oh
 
              5 | throw("uh oh")
                  ^^^^^^^^^^^^^^
              at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
-      ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+
+      0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
           .trimIndent()
@@ -233,14 +236,15 @@ class CliTestRunnerTest {
         """
       module test
         examples
-          ❌ fail
+          ✘ fail
              –– Pkl Error ––
              uh oh
 
              5 | throw("uh oh")
                  ^^^^^^^^^^^^^^
              at test#examples["fail"][#1] (/tempDir/test.pkl, line xx)
-      ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+
+      0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
     """
           .trimIndent()
@@ -435,10 +439,11 @@ class CliTestRunnerTest {
         """
         module test
           examples
-            ❌ nums
+            ✘ nums
                (/tempDir/test.pkl, line xx)
                Output mismatch: Expected "nums" to contain 1 examples, but found 2
-        ❌ 0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
+
+        0.0% tests pass [1/1 failed], 0.0% asserts pass [1/1 failed]
 
         """
           .trimIndent()
@@ -474,6 +479,7 @@ class CliTestRunnerTest {
           module test
             examples
               ✍️ nums
+
           1 examples written
 
           """

pkl-commons-cli/src/main/kotlin/org/pkl/commons/cli/CliCommand.kt

@@ -164,6 +164,8 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
     )
   }
 
+  protected val useColor: Boolean by lazy { cliOptions.color?.hasColor() ?: false }
+
   private val proxyAddress by lazy {
     cliOptions.httpProxy
       ?: project?.evaluatorSettings?.http?.proxy?.address ?: settings.http?.proxy?.address
@@ -284,7 +286,7 @@ abstract class CliCommand(protected val cliOptions: CliBaseOptions) {
       .setEnvironmentVariables(environmentVariables)
       .addModuleKeyFactories(moduleKeyFactories(modulePathResolver))
       .addResourceReaders(resourceReaders(modulePathResolver))
-      .setColor(cliOptions.color?.hasColor() ?: false)
+      .setColor(useColor)
       .setLogger(Loggers.stdErr())
       .setTimeout(cliOptions.timeout)
       .setModuleCacheDir(moduleCacheDir)

pkl-core/src/main/java/org/pkl/core/EvaluatorImpl.java

@@ -235,7 +235,7 @@ public TestResults evaluateTest(ModuleSource moduleSource, boolean overwrite) {
     return doEvaluate(
         moduleSource,
         (module) -> {
-          var testRunner = new TestRunner(logger, frameTransformer, overwrite);
+          var testRunner = new TestRunner(logger, frameTransformer, overwrite, color);
           return testRunner.run(module);
         });
   }

pkl-core/src/main/java/org/pkl/core/TestResults.java

@@ -92,7 +92,7 @@ public boolean failed() {
    * being written.
    */
   public boolean isExampleWrittenFailure() {
-    if (!failed() || !examples.failed()) return false;
+    if (!failed() || facts.failed() || !examples.failed()) return false;
     for (var testResult : examples.results) {
       if (!testResult.isExampleWritten) {
         return false;
@@ -295,7 +295,19 @@ public TestResult build() {
     }
   }
 
+  /**
+   * Indicates that an exception was thrown when evaluating the assertion.
+   *
+   * @param message The message of the underlying exception.
+   * @param exception The exception thrown by Pkl
+   */
   public record Error(String message, PklException exception) {}
 
+  /**
+   * Indicates that an assertion failed.
+   *
+   * @param kind The type of assertion failure.
+   * @param message The detailed message for the failure.
+   */
   public record Failure(String kind, String message) {}
 }