From ad3dd570988c74d3d908b261a6bbeb1bd5c695d5 Mon Sep 17 00:00:00 2001 From: Colin Caine Date: Mon, 23 May 2022 10:45:33 +0100 Subject: [PATCH 1/2] Capture output per test (For most tests) --- Manifest.toml | 12 +++--- Project.toml | 5 +-- src/runner.jl | 38 ++++++++++++++++--- src/tojson.jl | 31 +++++++++++---- .../all_passing_with_debugging/results.json | 8 ++-- .../all_passing_with_debugging/runtests.jl | 16 +++++--- test/fixtures/complex-numbers/results.json | 10 ++--- test/fixtures/everything_at_once/results.json | 9 ++--- test/fixtures/nested/results.json | 8 ++-- test/fixtures/one_fail/results.json | 2 +- test/fixtures/truncated_output/results.json | 2 +- test/fixtures/truncated_output/runtests.jl | 4 +- 12 files changed, 95 insertions(+), 50 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 6fd5841..6fe862a 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -1,5 +1,6 @@ # This file is machine-generated - editing it directly is not advised +julia_version = "1.7.2" manifest_format = "2.0" [[deps.Base64]] @@ -9,6 +10,12 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +[[deps.IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.2" + [[deps.InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -49,11 +56,6 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -[[deps.Suppressor]] -git-tree-sha1 = "c6ed566db2fe3931292865b966d6d140b7ef32a9" -uuid = "fd094767-a336-5f1f-9728-57cf17d0bbfb" -version = "0.2.1" - [[deps.Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/Project.toml b/Project.toml index d530b6b..de45b04 100644 --- a/Project.toml +++ b/Project.toml @@ -4,18 +4,17 @@ authors = ["Sascha Mann ", "Exercism", "Contributors"] version = "0.1.0" [deps] +IOCapture = "b5f81e59-6552-4d32-b1f0-c071b021bf89" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] JSON = "0.21" -Suppressor = "0.2" julia = "1.7" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "InteractiveUtils"] diff --git a/src/runner.jl b/src/runner.jl index 876bbc0..84bc5f7 100644 --- a/src/runner.jl +++ b/src/runner.jl @@ -1,6 +1,32 @@ -using Suppressor +using IOCapture using Test +const test_outputs = Dict() + +macro test(expr) + f(result, output) = test_outputs[result] = (output=output, expr=expr); + :( + c = $IOCapture.capture() do + $Test.@test $(esc(expr)); + end; + $f(c.value, c.output); + c.value + ) +end + +macro test_throws(extype, expr) + f(result, output) = test_outputs[result] = (output=output, expr=expr); + :( + c = $IOCapture.capture() do + $Test.@test_throws $extype $(esc(expr)); + end; + $f(c.value, c.output); + c.value + ) +end + +# test_logs is difficult. + """ runtests(testfile) @@ -8,11 +34,11 @@ Wrap the testfile in a ReportingTestSet and capture all output. Returns the output and ReportingTestSet. """ function runtests(testfile) - # The Suppressor macro wraps everything in a try...finally block, therefore rts needs to be introduced before - local rts - output = @capture_out rts = @testset ReportingTestSet "" begin - include(testfile) + c = IOCapture.capture() do + @testset ReportingTestSet "" begin + include(testfile) + end end - output, rts + c.output, c.value end diff --git a/src/tojson.jl b/src/tojson.jl index 315699d..b09d548 100644 --- a/src/tojson.jl +++ b/src/tojson.jl @@ -10,7 +10,7 @@ const MAX_REPORTED_FAILURES_PER_TESTSET = 5 const MAX_REPORTED_PASSING_TEST_CODE_PER_COLLAPSE = 5 """ - tojson(output::String, ts::ReportingTestSet) + tojson(general_output::String, ts::ReportingTestSet) Takes user output and a ReportingTestSet and converts it to a JSON string as expected by the interface. @@ -56,7 +56,7 @@ For more information, check the reference: https://github.com/exercism/docs/blob/main/building/tooling/test-runners/interface.md """ # TODO: Capture output per-test -function tojson(output::String, ts::ReportingTestSet) +function tojson(general_output::String, ts::ReportingTestSet) if length(ts.results) == 1 && ts.results[1] isa Test.Error # There has been a syntax error or similar and no tests have run. # Otherwise ts.results[1] will be a ReportingTestSet. @@ -83,20 +83,29 @@ function tojson(output::String, ts::ReportingTestSet) # Flag set in push_result!(), used for top-level status property any_failed = false # All stdout from the top level test set, used for all tests. - output = truncate_output(output) + general_output = truncate_output(general_output) + + unescape(e) = e + unescape(e::String) = startswith(e, "\$(") ? unescape(Meta.parse(e[3:end-1])) : e + unescape(e::Expr) = e.head == :escape ? unescape(e.args[1]) : e function test_code(result::Test.Result) + if haskey(test_outputs, result) + expr = test_outputs[result].expr + else + expr = unescape(result.orig_expr) + end if hasproperty(result, :test_type) && startswith(string(result.test_type), "test_throws") - "@test_throws $(result.data) $(result.orig_expr)" + "@test_throws $(result.data) $expr" elseif result isa Test.LogTestFailure - "@test_logs $(join(result.patterns, ' ')) $(result.orig_expr)" + "@test_logs $(join(result.patterns, ' ')) $expr" elseif hasproperty(result, :test_type) && result.test_type == :test_unbroken - "@test_broken $(result.orig_expr)" + "@test_broken $expr" elseif result isa Test.Broken macro_name = result.test_type === :skipped ? "@test_skip " : "@test_broken " - "$macro_name $(result.orig_expr)" + "$macro_name $expr" else - "@test $(result.orig_expr)" + "@test $expr" end end @@ -132,6 +141,12 @@ function tojson(output::String, ts::ReportingTestSet) any_failed = any_failed || status in ("fail", "error") + if haskey(test_outputs, result) + output = truncate_output(test_outputs[result].output) + else + output = general_output + end + return push!(tests, Dict(filter( ((k, v),) -> !isnothing(v), ( "name" => name, "status" => status, diff --git a/test/fixtures/all_passing_with_debugging/results.json b/test/fixtures/all_passing_with_debugging/results.json index 90da02d..a693bff 100644 --- a/test/fixtures/all_passing_with_debugging/results.json +++ b/test/fixtures/all_passing_with_debugging/results.json @@ -4,15 +4,15 @@ "tests": [ { "name": "first test", - "test_code": "@test x == 1", + "test_code": "@test f1(1) == 1", "status": "pass", - "output": "x = 1\nI'M HERE!\n" + "output": "x = 1\n" }, { "name": "second test", - "test_code": "@test 2 == 2", + "test_code": "@test f2(2) == 2", "status": "pass", - "output": "x = 1\nI'M HERE!\n" + "output": "I'M HERE!\n" } ] } diff --git a/test/fixtures/all_passing_with_debugging/runtests.jl b/test/fixtures/all_passing_with_debugging/runtests.jl index 4017020..9528c5d 100644 --- a/test/fixtures/all_passing_with_debugging/runtests.jl +++ b/test/fixtures/all_passing_with_debugging/runtests.jl @@ -1,12 +1,18 @@ using Test +function f1(x) + @show x # debugging +end + +function f2(x) + println("I'M HERE!") + x +end + @testset "first test" begin - x = 1 - @show x # Debugging - @test x == 1 + @test f1(1) == 1 end @testset "second test" begin - println("I'M HERE!") - @test 2 == 2 + @test f2(2) == 2 end diff --git a/test/fixtures/complex-numbers/results.json b/test/fixtures/complex-numbers/results.json index a321426..d9d4ef6 100644 --- a/test/fixtures/complex-numbers/results.json +++ b/test/fixtures/complex-numbers/results.json @@ -3,12 +3,12 @@ "version": 2, "tests": [ { - "name": "ComplexNumber <: Number", + "name": "$(Expr(:escape, :(ComplexNumber <: Number)))", "test_code": "@test ComplexNumber <: Number", "status": "pass" }, { - "name": "ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)", + "name": "$(Expr(:escape, :(ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0))))", "test_code": "@test ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)", "status": "pass" }, @@ -97,19 +97,19 @@ "name": "Syntax sugar jm.5", "test_code": "@test false", "status": "fail", - "message": "Test Failed at ./runtests.jl:77\n Expression: false" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, false))" }, { "name": "Syntax sugar jm.6", "test_code": "@test false", "status": "fail", - "message": "Test Failed at ./runtests.jl:77\n Expression: false" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, false))" }, { "name": "Syntax sugar jm.7", "test_code": "@test false", "status": "fail", - "message": "Test Failed at ./runtests.jl:77\n Expression: false" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, false))" } ] } diff --git a/test/fixtures/everything_at_once/results.json b/test/fixtures/everything_at_once/results.json index e70cac2..3232075 100644 --- a/test/fixtures/everything_at_once/results.json +++ b/test/fixtures/everything_at_once/results.json @@ -5,22 +5,19 @@ { "name": "first test", "test_code": "@test x == 1", - "status": "pass", - "output": "x = 1\n" + "status": "pass" }, { "name": "second test", "test_code": "@test 1 == 2", "status": "fail", - "message": "Test Failed at ./runtests.jl:10\n Expression: 1 == 2\n Evaluated: 1 == 2", - "output": "x = 1\n" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(1 == 2)))" }, { "name": "third test", "test_code": "@test error(\"\")", "status": "error", - "message": "", - "output": "x = 1\n" + "message": "" } ] } diff --git a/test/fixtures/nested/results.json b/test/fixtures/nested/results.json index b5bba6e..6b689d8 100644 --- a/test/fixtures/nested/results.json +++ b/test/fixtures/nested/results.json @@ -11,19 +11,19 @@ "name": "outer » inner 1.2", "test_code": "@test 1 == 2", "status": "fail", - "message": "Test Failed at ./runtests.jl:8\n Expression: 1 == 2\n Evaluated: 1 == 2" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(1 == 2)))" }, { "name": "outer » inner 2", "test_code": "@test 3 == 4", "status": "fail", - "message": "Test Failed at ./runtests.jl:12\n Expression: 3 == 4\n Evaluated: 3 == 4" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(3 == 4)))" }, { "name": "outer 2 » inner 3", "test_code": "@test 5 == 6", "status": "fail", - "message": "Test Failed at ./runtests.jl:20\n Expression: 5 == 6\n Evaluated: 5 == 6" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(5 == 6)))" }, { "name": "outer 2", @@ -45,7 +45,7 @@ "name": "outer 3 » inner 4", "test_code": "@test f(5) == 6", "status": "fail", - "message": "Test Failed at ./runtests.jl:32\n Expression: f(5) == 6\n Evaluated: 5 == 6" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(f(5) == 6)))" } ] } diff --git a/test/fixtures/one_fail/results.json b/test/fixtures/one_fail/results.json index 1f0ddb1..6b7c532 100644 --- a/test/fixtures/one_fail/results.json +++ b/test/fixtures/one_fail/results.json @@ -11,7 +11,7 @@ "name": "second test", "test_code": "@test x == 1", "status": "fail", - "message": "Test Failed at ./runtests.jl:9\n Expression: x == 1\n Evaluated: 2 == 1" + "message": "Test Failed at /home/colin/projects/exercism/repos/julia-test-runner/src/runner.jl:10\n Expression: $(Expr(:escape, :(x == 1)))" }, { "name": "third test", diff --git a/test/fixtures/truncated_output/results.json b/test/fixtures/truncated_output/results.json index e8ef2f0..d1f33b1 100644 --- a/test/fixtures/truncated_output/results.json +++ b/test/fixtures/truncated_output/results.json @@ -4,7 +4,7 @@ "tests": [ { "name": "first test", - "test_code": "@test 1 == 1", + "test_code": "@test f(1) == 1", "status": "pass", "output": "Output was truncated. Please limit to 500 chars\n\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" } diff --git a/test/fixtures/truncated_output/runtests.jl b/test/fixtures/truncated_output/runtests.jl index 7955110..5f0445b 100644 --- a/test/fixtures/truncated_output/runtests.jl +++ b/test/fixtures/truncated_output/runtests.jl @@ -1,6 +1,6 @@ using Test @testset "first test" begin - println("x"^5000) - @test 1 == 1 + f(x) = (println("x"^5000); x) + @test f(1) == 1 end From 4413802985afa16f69c74ca4c7d41b671e1f34c1 Mon Sep 17 00:00:00 2001 From: Colin Caine Date: Mon, 23 May 2022 10:56:37 +0100 Subject: [PATCH 2/2] Restore debugging in everything_at_once --- test/fixtures/everything_at_once/results.json | 5 +++-- test/fixtures/everything_at_once/runtests.jl | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/fixtures/everything_at_once/results.json b/test/fixtures/everything_at_once/results.json index 3232075..c08c96c 100644 --- a/test/fixtures/everything_at_once/results.json +++ b/test/fixtures/everything_at_once/results.json @@ -4,8 +4,9 @@ "tests": [ { "name": "first test", - "test_code": "@test x == 1", - "status": "pass" + "test_code": "@test f(1) == 1", + "status": "pass", + "output": "x = 1\n" }, { "name": "second test", diff --git a/test/fixtures/everything_at_once/runtests.jl b/test/fixtures/everything_at_once/runtests.jl index 9a6cd85..8dbcd4c 100644 --- a/test/fixtures/everything_at_once/runtests.jl +++ b/test/fixtures/everything_at_once/runtests.jl @@ -1,9 +1,8 @@ using Test @testset "first test" begin - x = 1 - @show x # Debugging - @test x == 1 + f(x) = @show x + @test f(1) == 1 end @testset "second test" begin