Draft: Capture output per test

Manifest.toml

@@ -1,5 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
+julia_version = "1.7.2"
 manifest_format = "2.0"
 
 [[deps.Base64]]
@@ -9,6 +10,12 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 deps = ["Printf"]
 uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
 
+[[deps.IOCapture]]
+deps = ["Logging", "Random"]
+git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a"
+uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
+version = "0.2.2"
+
 [[deps.InteractiveUtils]]
 deps = ["Markdown"]
 uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
@@ -49,11 +56,6 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 [[deps.Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
-[[deps.Suppressor]]
-git-tree-sha1 = "c6ed566db2fe3931292865b966d6d140b7ef32a9"
-uuid = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
-version = "0.2.1"
-
 [[deps.Test]]
 deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

Project.toml

@@ -4,18 +4,17 @@ authors = ["Sascha Mann <[email protected]>", "Exercism", "Contributors"]
 version = "0.1.0"
 
 [deps]
+IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 JSON = "0.21"
-Suppressor = "0.2"
 julia = "1.7"
 
 [extras]
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
 test = ["Test", "InteractiveUtils"]

src/runner.jl

@@ -1,18 +1,44 @@
-using Suppressor
+using IOCapture
 using Test
 
+const test_outputs = Dict()
+
+macro test(expr)
+    f(result, output) = test_outputs[result] = (output=output, expr=expr);
+    :(
+      c = $IOCapture.capture() do
+          $Test.@test $(esc(expr));
+      end;
+      $f(c.value, c.output);
+      c.value
+     )
+end
+
+macro test_throws(extype, expr)
+    f(result, output) = test_outputs[result] = (output=output, expr=expr);
+    :(
+      c = $IOCapture.capture() do
+          $Test.@test_throws $extype $(esc(expr));
+      end;
+      $f(c.value, c.output);
+      c.value
+     )
+end
+
+# test_logs is difficult.
+
 """
     runtests(testfile)
 
 Wrap the testfile in a ReportingTestSet and capture all output.
 Returns the output and ReportingTestSet.
 """
 function runtests(testfile)
-    # The Suppressor macro wraps everything in a try...finally block, therefore rts needs to be introduced before
-    local rts
-    output = @capture_out rts = @testset ReportingTestSet "" begin
-        include(testfile)
+    c = IOCapture.capture() do
+        @testset ReportingTestSet "" begin
+            include(testfile)
+        end
     end
 
-    output, rts
+    c.output, c.value
 end

src/tojson.jl

@@ -10,7 +10,7 @@ const MAX_REPORTED_FAILURES_PER_TESTSET = 5
 const MAX_REPORTED_PASSING_TEST_CODE_PER_COLLAPSE = 5
 
 """
-    tojson(output::String, ts::ReportingTestSet)
+    tojson(general_output::String, ts::ReportingTestSet)
 
 Takes user output and a ReportingTestSet and converts it to a JSON string as
 expected by the interface.
@@ -56,7 +56,7 @@ For more information, check the reference:
 https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md
 """
 # TODO: Capture output per-test
-function tojson(output::String, ts::ReportingTestSet)
+function tojson(general_output::String, ts::ReportingTestSet)
     if length(ts.results) == 1 && ts.results[1] isa Test.Error
         # There has been a syntax error or similar and no tests have run.
         # Otherwise ts.results[1] will be a ReportingTestSet.
@@ -83,20 +83,29 @@ function tojson(output::String, ts::ReportingTestSet)
     # Flag set in push_result!(), used for top-level status property
     any_failed = false
     # All stdout from the top level test set, used for all tests.
-    output = truncate_output(output)
+    general_output = truncate_output(general_output)
+
+    unescape(e) = e
+    unescape(e::String) = startswith(e, "\$(") ? unescape(Meta.parse(e[3:end-1])) : e
+    unescape(e::Expr) = e.head == :escape ? unescape(e.args[1]) : e
 
     function test_code(result::Test.Result)
+        if haskey(test_outputs, result)
+            expr = test_outputs[result].expr
+        else
+            expr = unescape(result.orig_expr)
+        end
         if hasproperty(result, :test_type) && startswith(string(result.test_type), "test_throws")
-            "@test_throws $(result.data) $(result.orig_expr)"
+            "@test_throws $(result.data) $expr"
         elseif result isa Test.LogTestFailure
-            "@test_logs $(join(result.patterns, ' ')) $(result.orig_expr)"
+            "@test_logs $(join(result.patterns, ' ')) $expr"
         elseif hasproperty(result, :test_type) && result.test_type == :test_unbroken
-            "@test_broken $(result.orig_expr)"
+            "@test_broken $expr"
         elseif result isa Test.Broken
             macro_name = result.test_type === :skipped ? "@test_skip " : "@test_broken "
-            "$macro_name $(result.orig_expr)"
+            "$macro_name $expr"
         else
-            "@test $(result.orig_expr)"
+            "@test $expr"
         end
     end
 
@@ -132,6 +141,12 @@ function tojson(output::String, ts::ReportingTestSet)
 
         any_failed = any_failed || status in ("fail", "error")
 
+        if haskey(test_outputs, result)
+            output = truncate_output(test_outputs[result].output)
+        else
+            output = general_output
+        end
+
         return push!(tests, Dict(filter( ((k, v),) -> !isnothing(v), (
             "name" => name,
             "status" => status,

test/fixtures/all_passing_with_debugging/results.json

@@ -4,15 +4,15 @@
     "tests": [
         {
             "name": "first test",
-            "test_code": "@test x == 1",
+            "test_code": "@test f1(1) == 1",
             "status": "pass",
-            "output": "x = 1\nI'M HERE!\n"
+            "output": "x = 1\n"
         },
         {
             "name": "second test",
-            "test_code": "@test 2 == 2",
+            "test_code": "@test f2(2) == 2",
             "status": "pass",
-            "output": "x = 1\nI'M HERE!\n"
+            "output": "I'M HERE!\n"
         }
     ]
 }

test/fixtures/all_passing_with_debugging/runtests.jl

@@ -1,12 +1,18 @@
 using Test
 
+function f1(x)
+    @show x # debugging
+end
+
+function f2(x)
+    println("I'M HERE!")
+    x
+end
+
 @testset "first test" begin
-    x = 1
-    @show x # Debugging
-    @test x == 1
+    @test f1(1) == 1
 end
 
 @testset "second test" begin
-    println("I'M HERE!")
-    @test 2 == 2
+    @test f2(2) == 2
 end

test/fixtures/complex-numbers/results.json

@@ -3,12 +3,12 @@
     "version": 2,
     "tests": [
         {
-            "name": "ComplexNumber <: Number",
+            "name": "$(Expr(:escape, :(ComplexNumber <: Number)))",
             "test_code": "@test ComplexNumber <: Number",
             "status": "pass"
         },
         {
-            "name": "ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)",
+            "name": "$(Expr(:escape, :(ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0))))",
             "test_code": "@test ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)",
             "status": "pass"
         },
@@ -97,19 +97,19 @@
             "name": "Syntax sugar jm.5",
             "test_code": "@test false",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:77\n  Expression: false"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, false))"
         },
         {
             "name": "Syntax sugar jm.6",
             "test_code": "@test false",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:77\n  Expression: false"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, false))"
         },
         {
             "name": "Syntax sugar jm.7",
             "test_code": "@test false",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:77\n  Expression: false"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, false))"
         }
     ]
 }

test/fixtures/everything_at_once/results.json

@@ -4,23 +4,21 @@
     "tests": [
         {
             "name": "first test",
-            "test_code": "@test x == 1",
+            "test_code": "@test f(1) == 1",
             "status": "pass",
             "output": "x = 1\n"
         },
         {
             "name": "second test",
             "test_code": "@test 1 == 2",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:10\n  Expression: 1 == 2\n   Evaluated: 1 == 2",
-            "output": "x = 1\n"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, :(1 == 2)))"
         },
         {
             "name": "third test",
             "test_code": "@test error(\"\")",
             "status": "error",
-            "message": "",
-            "output": "x = 1\n"
+            "message": ""
         }
     ]
 }

test/fixtures/everything_at_once/runtests.jl

@@ -1,9 +1,8 @@
 using Test
 
 @testset "first test" begin
-    x = 1
-    @show x # Debugging
-    @test x == 1
+    f(x) = @show x
+    @test f(1) == 1
 end
 
 @testset "second test" begin

test/fixtures/nested/results.json

@@ -11,19 +11,19 @@
             "name": "outer » inner 1.2",
             "test_code": "@test 1 == 2",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:8\n  Expression: 1 == 2\n   Evaluated: 1 == 2"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, :(1 == 2)))"
         },
         {
             "name": "outer » inner 2",
             "test_code": "@test 3 == 4",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:12\n  Expression: 3 == 4\n   Evaluated: 3 == 4"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, :(3 == 4)))"
         },
         {
             "name": "outer 2 » inner 3",
             "test_code": "@test 5 == 6",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:20\n  Expression: 5 == 6\n   Evaluated: 5 == 6"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, :(5 == 6)))"
         },
         {
             "name": "outer 2",
@@ -45,7 +45,7 @@
             "name": "outer 3 » inner 4",
             "test_code": "@test f(5) == 6",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:32\n  Expression: f(5) == 6\n   Evaluated: 5 == 6"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, :(f(5) == 6)))"
         }
     ]
 }

test/fixtures/one_fail/results.json

@@ -11,7 +11,7 @@
             "name": "second test",
             "test_code": "@test x == 1",
             "status": "fail",
-            "message": "Test Failed at ./runtests.jl:9\n  Expression: x == 1\n   Evaluated: 2 == 1"
+            "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n  Expression: $(Expr(:escape, :(x == 1)))"
         },
         {
             "name": "third test",

test/fixtures/truncated_output/results.json

@@ -4,7 +4,7 @@
     "tests": [
         {
             "name": "first test",
-            "test_code": "@test 1 == 1",
+            "test_code": "@test f(1) == 1",
             "status": "pass",
             "output": "Output was truncated. Please limit to 500 chars\n\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
         }

test/fixtures/truncated_output/runtests.jl

@@ -1,6 +1,6 @@
 using Test
 
 @testset "first test" begin
-    println("x"^5000)
-    @test 1 == 1
+    f(x) = (println("x"^5000); x)
+    @test f(1) == 1
 end