diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60f3afe..0d22e2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: # Change this to add or remove tests env: - TESTS: all_passing,all_passing_with_debugging,everything_at_once,one_fail,one_passing,truncated_output + TESTS: all_passing,all_passing_with_debugging,everything_at_once,one_fail,one_passing,truncated_output,grains,complex-numbers JULIA_NUM_THREADS: 2 steps: @@ -40,8 +40,10 @@ jobs: # Create container run(`docker create --name jtr-$test jtr-img irrelevant /tmp/ /tmp/`) - # Copy test file to container + # Copy test file(s) to container run(`docker cp "test/fixtures/$test/runtests.jl" jtr-$test:/tmp/`) + isfile("test/fixtures/$test/$test.jl") && + run(`docker cp "test/fixtures/$test/$test.jl" jtr-$test:/tmp/`) # Start container to run the test runner run(`docker start -a jtr-$test`) @@ -61,11 +63,23 @@ jobs: const tests = split(ENV["TESTS"], ",") + # Shorten the error messages so we don't need to care about specific paths, file numbers, etc. + function shorten_messages!(result) + if result["status"] == "error" && haskey(result, "message") + result["message"] = split(result["message"], '\n')[1] + end + end + @testset "Compare results.json files" begin @testset "$test" for test in tests expected = JSON.parsefile(joinpath("test", "fixtures", "$test", "results.json")) created = JSON.parsefile(joinpath("results", "$test.json")) + shorten_messages!(expected) + shorten_messages!(created) + foreach(shorten_messages!, expected["tests"]) + foreach(shorten_messages!, created["tests"]) + @test expected == created end end diff --git a/src/tojson.jl b/src/tojson.jl index 43b786d..9abfd93 100644 --- a/src/tojson.jl +++ b/src/tojson.jl @@ -1,6 +1,13 @@ using Test using JSON +""" +Testsets with more than this number of tests will have all of their passing +tests collapsed into a single report. +""" +const TEST_RESULT_COLLAPSE_THRESHOLD = 2 +const MAX_REPORTED_FAILURES_PER_TESTSET = 5 + """ tojson(output::String, ts::ReportingTestSet) @@ -72,12 +79,61 @@ function tojson(output::String, ts::ReportingTestSet) end end - # Flag set in walk!(), used for top-level status property + # Flag set in push_result!(), used for top-level status property any_failed = false + # All stdout from the top level test set, used for all tests. output = truncate_output(output) """ - walk!(prefix, tests, testset) + push_result!(tests, result, name) + + Push a test result called `name` to `tests`. Returns the pushed Dict or `nothing` if the test was not pushed. + """ + function push_result!(tests, result::Test.Result, name) + status = nothing + message = nothing + test_code = nothing + + if result isa Test.Pass + status = "pass" + test_code = "@test $(result.orig_expr)" + elseif result isa Test.Fail + status = "fail" + message = string(result) + test_code = "@test $(result.orig_expr)" + elseif result isa Test.LogTestFailure + status = "fail" + message = string(result) + test_code = "@test_logs $(join(result.patterns, ' ')) $(result.orig_expr)" + elseif result isa Test.Error + status = "error" + message = result.backtrace + test_code = "@test " * result.orig_expr + elseif result isa Test.Broken + if result.test_type === :skipped + return nothing + end + # TODO: In the future we might have a new `status = skip` + message = string(result) + test_code = result.test_type === :skipped ? "@test_skip " : "@test_broken " + test_code *= string(result.orig_expr) + else + error("Unknown testset.results item: $result") + end + + any_failed = any_failed || status in ("fail", "error") + + return push!(tests, Dict(filter( ((k, v),) -> !isnothing(v), ( + "name" => name, + "status" => status, + "message" => message, + "test_code" => test_code, + "output" => output, + )))) + end + + """ + walk!(tests, prefix, testset) Walk the tree of testsets, pushing Dicts to `tests` describing each test result. Returns nothing. @@ -85,56 +141,47 @@ function tojson(output::String, ts::ReportingTestSet) function walk!(tests, prefix, testset) name = isempty(prefix) ? testset.description : "$prefix » $(testset.description)" - # Should we number the tests? - number_tests = count(x -> x isa Test.Result, testset.results) > 1 + num_results = count(x -> x isa Test.Result, testset.results) - for (n, result) in enumerate(testset.results) - status = nothing - message = nothing - test_code = nothing - - if result isa Test.Pass - status = "pass" - message = nothing - elseif result isa Test.Fail - status = "fail" - message = string(result) - test_code = "@test " * result.orig_expr - elseif result isa Test.LogTestFailure - status = "fail" - message = string(result) - test_code = "@test_logs $(join(result.patterns, ' ')) $(result.orig_expr)" - elseif result isa Test.Error - status = "error" - message = result.backtrace - test_code = "@test " * result.orig_expr - elseif result isa Test.Broken - if result.test_type === :skipped - continue - end - # TODO: In the future we might have a new `status = skip` - message = string(result) - test_code = result.test_type === :skipped ? "@test_skip " : "@test_broken " - test_code *= string(result.orig_expr) - elseif result isa Test.AbstractTestSet - # Descend into nested test sets - child_has_failed = walk!(tests, name, result) - continue + function test_name(result, idx) + if name == "" # Tests that aren't in a testset + string(result.orig_expr) else - error("Unknown testset.results item: $result") + # If there's more than one test in the testset, distinguish + # them with numbers. + num_results > 1 ? "$name.$idx" : name end + end - any_failed = any_failed || status in ("fail", "error") - - push!(tests, Dict(filter( ((k, v),) -> !isnothing(v), ( - "name" => number_tests ? "$name.$n" : name, - "status" => status, - "message" => message, - "test_code" => test_code, - "output" => output, - )))) + # Should we collapse all passing tests into one report? + passing_tests = filter(x -> x isa Test.Pass, testset.results) + num_passing = length(passing_tests) + collapse_passing_tests = num_results >= TEST_RESULT_COLLAPSE_THRESHOLD && num_passing > 1 && name != "" + + if collapse_passing_tests + test_code = join(("@test $(r.orig_expr)" for r in passing_tests), '\n') + collapsed_name = num_passing == num_results ? name : "$name » $num_passing tests" + push!(tests, Dict( + "name" => collapsed_name, + "status" => "pass", + "test_code" => test_code + )) end + # Push up to MAX_REPORTED_FAILURES_PER_TESTSET failing test results. + # Also recurse into any test sets and report passing test results if + # they weren't collapsed. + num_reported_failures = 0 + for (n, result) in enumerate(testset.results) + if result isa Test.AbstractTestSet + walk!(tests, name, result) + elseif result isa Test.Pass + collapse_passing_tests || push_result!(tests, result, test_name(result, n)) + else + num_reported_failures >= MAX_REPORTED_FAILURES_PER_TESTSET && continue + num_reported_failures += !isnothing(push_result!(tests, result, test_name(result, n))) + end + end return nothing end diff --git a/test/fixtures/all_passing/results.json b/test/fixtures/all_passing/results.json index 5e6efa7..c221b44 100644 --- a/test/fixtures/all_passing/results.json +++ b/test/fixtures/all_passing/results.json @@ -4,14 +4,17 @@ "tests": [ { "name": "first test", + "test_code": "@test x == 1", "status": "pass" }, { "name": "second test", + "test_code": "@test 2 == 2", "status": "pass" }, { "name": "third test", + "test_code": "@test 3 == 3", "status": "pass" } ] diff --git a/test/fixtures/all_passing_with_debugging/results.json b/test/fixtures/all_passing_with_debugging/results.json index c623422..90da02d 100644 --- a/test/fixtures/all_passing_with_debugging/results.json +++ b/test/fixtures/all_passing_with_debugging/results.json @@ -4,11 +4,13 @@ "tests": [ { "name": "first test", + "test_code": "@test x == 1", "status": "pass", "output": "x = 1\nI'M HERE!\n" }, { "name": "second test", + "test_code": "@test 2 == 2", "status": "pass", "output": "x = 1\nI'M HERE!\n" } diff --git a/test/fixtures/complex-numbers/complex-numbers.jl b/test/fixtures/complex-numbers/complex-numbers.jl new file mode 100644 index 0000000..51dc110 --- /dev/null +++ b/test/fixtures/complex-numbers/complex-numbers.jl @@ -0,0 +1,59 @@ +import Base: ==, +, -, *, /, abs, conj, exp, imag, real, zero, promote_rule + +struct ComplexNumber{T <: Real} <: Number + a::T + b::T +end + +function ComplexNumber(a::T, b::U) where {T <: Real, U <: Real} + V = promote_type(T, U) + ComplexNumber{V}(V(a), V(b)) +end + +# This makes isapprox work and is needed for Bonus 2. +ComplexNumber{T}(a::Real) where T = ComplexNumber{T}(T(a), zero(T)) + +# Convert ComplexNumber{U} to ComplexNumber{T}, required for Bonus 2. +ComplexNumber{T}(a::ComplexNumber) where T = ComplexNumber{T}(T(a.a), T(a.b)) + +# You can make isapprox work without defining the constructor above if you +# define zero and real +zero(::Type{ComplexNumber{T}}) where T = ComplexNumber(zero(T), zero(T)) +zero(x::ComplexNumber{T}) where T = ComplexNumber(zero(T), zero(T)) + +real(::Type{ComplexNumber{T}}) where T = T +real(x::ComplexNumber) = x.a + +imag(::Type{ComplexNumber{T}}) where T = T +imag(x::ComplexNumber) = x.b + +# Default structural equality method uses ===, but we will often contain +# floats, which must be compared with == +==(x::ComplexNumber, y::ComplexNumber) = x.a == y.a && x.b == y.b + +conj(x::ComplexNumber) = ComplexNumber(x.a, -x.b) +abs(x::ComplexNumber) = sqrt(x.a * x.a + x.b * x.b) + +-(x::ComplexNumber) = ComplexNumber(-x.a, -x.b) ++(x::ComplexNumber, y::ComplexNumber) = ComplexNumber(x.a + y.a, x.b + y.b) +-(x::ComplexNumber, y::ComplexNumber) = x + -y + +*(x::ComplexNumber, y::ComplexNumber) = ComplexNumber(x.a * y.a - x.b * y.b, + x.b * y.a + x.a * y.b) + +/(x::ComplexNumber, y::ComplexNumber) = ComplexNumber((x.a * y.a + x.b * y.b) / (y.a^2 + y.b^2), + (x.b * y.a - x.a * y.b) / (y.a^2 + y.b^2)) + +# Bonus 1: exp + +# Promotion rule makes mixed Real/Complex mathematical operations work + +# Commenting this out makes some of the tests fail, which is useful for testing the test runner :) +# promote_rule(::Type{T}, ::Type{ComplexNumber{U}}) where {T <: Real, U} = ComplexNumber{promote_type(T, U)} +exp(x::ComplexNumber) = ℯ^x.a * ComplexNumber(cos(x.b), sin(x.b)) + +# Bonus 2: jm unit + +# Better to use false here because we need that strong multiplicative zero +# to make `0 + NaN * jm === ComplexNumber(0, NaN)` +const jm = ComplexNumber(false, true) diff --git a/test/fixtures/complex-numbers/results.json b/test/fixtures/complex-numbers/results.json new file mode 100644 index 0000000..a321426 --- /dev/null +++ b/test/fixtures/complex-numbers/results.json @@ -0,0 +1,115 @@ +{ + "status": "fail", + "version": 2, + "tests": [ + { + "name": "ComplexNumber <: Number", + "test_code": "@test ComplexNumber <: Number", + "status": "pass" + }, + { + "name": "ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)", + "test_code": "@test ComplexNumber(0, 1) ^ 2 == ComplexNumber(-1, 0)", + "status": "pass" + }, + { + "name": "Arithmetic » Addition", + "test_code": "@test ComplexNumber(1, 0) + ComplexNumber(2, 0) == ComplexNumber(3, 0)\n@test ComplexNumber(0, 1) + ComplexNumber(0, 2) == ComplexNumber(0, 3)\n@test ComplexNumber(1, 2) + ComplexNumber(3, 4) == ComplexNumber(4, 6)", + "status": "pass" + }, + { + "name": "Arithmetic » Subtraction", + "test_code": "@test ComplexNumber(1, 0) - ComplexNumber(2, 0) == ComplexNumber(-1, 0)\n@test ComplexNumber(0, 1) - ComplexNumber(0, 2) == ComplexNumber(0, -1)\n@test ComplexNumber(1, 2) - ComplexNumber(3, 4) == ComplexNumber(-2, -2)", + "status": "pass" + }, + { + "name": "Arithmetic » Multiplication", + "test_code": "@test ComplexNumber(1, 0) * ComplexNumber(2, 0) == ComplexNumber(2, 0)\n@test ComplexNumber(0, 1) * ComplexNumber(0, 2) == ComplexNumber(-2, 0)\n@test ComplexNumber(1, 2) * ComplexNumber(3, 4) == ComplexNumber(-5, 10)", + "status": "pass" + }, + { + "name": "Arithmetic » Division", + "test_code": "@test ComplexNumber(1, 0) / ComplexNumber(2, 0) == ComplexNumber(0.5, 0)\n@test ComplexNumber(0, 1) / ComplexNumber(0, 2) == ComplexNumber(0.5, 0)\n@test ComplexNumber(1, 2) / ComplexNumber(3, 4) ≈ ComplexNumber(0.44, 0.08)", + "status": "pass" + }, + { + "name": "Absolute value", + "test_code": "@test abs(ComplexNumber(5, 0)) == 5\n@test abs(ComplexNumber(-5, 0)) == 5\n@test abs(ComplexNumber(0, 5)) == 5\n@test abs(ComplexNumber(0, -5)) == 5\n@test abs(ComplexNumber(3, 4)) == 5", + "status": "pass" + }, + { + "name": "Complex conjugate", + "test_code": "@test conj(ComplexNumber(5.0, 0.0)) == ComplexNumber(5.0, 0.0)\n@test conj(ComplexNumber(0, 5)) == ComplexNumber(0, -5)\n@test conj(ComplexNumber(1, 1)) == ComplexNumber(1, -1)", + "status": "pass" + }, + { + "name": "Real part", + "test_code": "@test real(ComplexNumber(1, 0)) == 1\n@test real(ComplexNumber(0, 1)) == 0\n@test real(ComplexNumber(1, 2)) == 1", + "status": "pass" + }, + { + "name": "Imaginary part", + "test_code": "@test imag(ComplexNumber(1, 0)) == 0\n@test imag(ComplexNumber(0, 1)) == 1\n@test imag(ComplexNumber(1, 2)) == 2", + "status": "pass" + }, + { + "name": "Complex exponential.1", + "test_code": "@test exp(ComplexNumber(0, π)) ≈ ComplexNumber(-1, 0)", + "status": "error", + "message": "promotion of types Float64 and ExercismTestReports.ComplexNumber{Float64} failed to change any arguments" + }, + { + "name": "Complex exponential.2", + "test_code": "@test exp(ComplexNumber(0, 0)) == ComplexNumber(1, 0)", + "status": "error", + "message": "promotion of types Float64 and ExercismTestReports.ComplexNumber{Float64} failed to change any arguments" + }, + { + "name": "Complex exponential.3", + "test_code": "@test exp(ComplexNumber(1, 0)) ≈ ComplexNumber(ℯ, 0)", + "status": "error", + "message": "promotion of types Float64 and ExercismTestReports.ComplexNumber{Float64} failed to change any arguments" + }, + { + "name": "Complex exponential.4", + "test_code": "@test exp(ComplexNumber(log(2), π)) ≈ ComplexNumber(-2, 0)", + "status": "error", + "message": "promotion of types Float64 and ExercismTestReports.ComplexNumber{Float64} failed to change any arguments" + }, + { + "name": "Syntax sugar jm » 2 tests", + "test_code": "@test ComplexNumber(0, 1) == jm\n@test ComplexNumber(-1, 0) == jm ^ 2", + "status": "pass" + }, + { + "name": "Syntax sugar jm.2", + "test_code": "@test ComplexNumber(1, 0) == 1 + 0jm", + "status": "error", + "message": "promotion of types Int64 and ExercismTestReports.ComplexNumber{Bool} failed to change any arguments" + }, + { + "name": "Syntax sugar jm.3", + "test_code": "@test ComplexNumber(1, 1) == 1 + 1jm", + "status": "error", + "message": "promotion of types Int64 and ExercismTestReports.ComplexNumber{Bool} failed to change any arguments" + }, + { + "name": "Syntax sugar jm.5", + "test_code": "@test false", + "status": "fail", + "message": "Test Failed at ./runtests.jl:77\n Expression: false" + }, + { + "name": "Syntax sugar jm.6", + "test_code": "@test false", + "status": "fail", + "message": "Test Failed at ./runtests.jl:77\n Expression: false" + }, + { + "name": "Syntax sugar jm.7", + "test_code": "@test false", + "status": "fail", + "message": "Test Failed at ./runtests.jl:77\n Expression: false" + } + ] +} diff --git a/test/fixtures/complex-numbers/runtests.jl b/test/fixtures/complex-numbers/runtests.jl new file mode 100644 index 0000000..ec9f107 --- /dev/null +++ b/test/fixtures/complex-numbers/runtests.jl @@ -0,0 +1,79 @@ +using Test + +include("complex-numbers.jl") + +@test ComplexNumber <: Number + +@test ComplexNumber(0, 1)^2 == ComplexNumber(-1, 0) + +@testset "Arithmetic" begin + @testset "Addition" begin + @test ComplexNumber(1, 0) + ComplexNumber(2, 0) == ComplexNumber(3, 0) + @test ComplexNumber(0, 1) + ComplexNumber(0, 2) == ComplexNumber(0, 3) + @test ComplexNumber(1, 2) + ComplexNumber(3, 4) == ComplexNumber(4, 6) + end + + @testset "Subtraction" begin + @test ComplexNumber(1, 0) - ComplexNumber(2, 0) == ComplexNumber(-1, 0) + @test ComplexNumber(0, 1) - ComplexNumber(0, 2) == ComplexNumber(0, -1) + @test ComplexNumber(1, 2) - ComplexNumber(3, 4) == ComplexNumber(-2, -2) + end + + @testset "Multiplication" begin + @test ComplexNumber(1, 0) * ComplexNumber(2, 0) == ComplexNumber(2, 0) + @test ComplexNumber(0, 1) * ComplexNumber(0, 2) == ComplexNumber(-2, 0) + @test ComplexNumber(1, 2) * ComplexNumber(3, 4) == ComplexNumber(-5, 10) + end + + @testset "Division" begin + @test ComplexNumber(1, 0) / ComplexNumber(2, 0) == ComplexNumber(0.5, 0) + @test ComplexNumber(0, 1) / ComplexNumber(0, 2) == ComplexNumber(0.5, 0) + @test ComplexNumber(1, 2) / ComplexNumber(3, 4) ≈ ComplexNumber(0.44, 0.08) + end +end + +@testset "Absolute value" begin + @test abs(ComplexNumber(5, 0)) == 5 + @test abs(ComplexNumber(-5, 0)) == 5 + @test abs(ComplexNumber(0, 5)) == 5 + @test abs(ComplexNumber(0, -5)) == 5 + @test abs(ComplexNumber(3, 4)) == 5 +end + +@testset "Complex conjugate" begin + @test conj(ComplexNumber(5.0, 0.0)) == ComplexNumber(5.0, 0.0) + @test conj(ComplexNumber(0, 5)) == ComplexNumber(0, -5) + @test conj(ComplexNumber(1, 1)) == ComplexNumber(1, -1) +end + +@testset "Real part" begin + @test real(ComplexNumber(1, 0)) == 1 + @test real(ComplexNumber(0, 1)) == 0 + @test real(ComplexNumber(1, 2)) == 1 +end + +@testset "Imaginary part" begin + @test imag(ComplexNumber(1, 0)) == 0 + @test imag(ComplexNumber(0, 1)) == 1 + @test imag(ComplexNumber(1, 2)) == 2 +end + +# Bonus A +@testset "Complex exponential" begin + @test exp(ComplexNumber(0, π)) ≈ ComplexNumber(-1, 0) + @test exp(ComplexNumber(0, 0)) == ComplexNumber(1, 0) + @test exp(ComplexNumber(1, 0)) ≈ ComplexNumber(ℯ, 0) + @test exp(ComplexNumber(log(2), π)) ≈ ComplexNumber(-2, 0) +end + +# Bonus B +@testset "Syntax sugar jm" begin + @test ComplexNumber(0, 1) == jm + @test ComplexNumber(1, 0) == 1 + 0jm + @test ComplexNumber(1, 1) == 1 + 1jm + @test ComplexNumber(-1, 0) == jm^2 + # More failing tests to show that only the first MAX_REPORTED_FAILURES are actually reported + for _ in 1:100 + @test false + end +end diff --git a/test/fixtures/everything_at_once/results.json b/test/fixtures/everything_at_once/results.json index 86cd8fe..e70cac2 100644 --- a/test/fixtures/everything_at_once/results.json +++ b/test/fixtures/everything_at_once/results.json @@ -4,6 +4,7 @@ "tests": [ { "name": "first test", + "test_code": "@test x == 1", "status": "pass", "output": "x = 1\n" }, @@ -18,7 +19,7 @@ "name": "third test", "test_code": "@test error(\"\")", "status": "error", - "message": "\nStacktrace:\n [1] error(s::String)\n @ Base ./error.jl:33\n [2] macro expansion\n @ /usr/local/julia/share/julia/stdlib/v1.7/Test/src/Test.jl:445 [inlined]\n [3] macro expansion\n @ ./runtests.jl:14 [inlined]\n [4] macro expansion\n @ /usr/local/julia/share/julia/stdlib/v1.7/Test/src/Test.jl:1283 [inlined]\n [5] top-level scope\n @ ./runtests.jl:14", + "message": "", "output": "x = 1\n" } ] diff --git a/test/fixtures/grains/grains.jl b/test/fixtures/grains/grains.jl new file mode 100644 index 0000000..726cf96 --- /dev/null +++ b/test/fixtures/grains/grains.jl @@ -0,0 +1,7 @@ +"""Calculate the number of grains on square `i`.""" +# 2^(i-1) +on_square(i) = i in 1:64 ? (UInt64(1) << (i-1)) : throw(DomainError(i)) + +"""Calculate the total number of grains on squares `1:i`""" +# (2^i) - 1 +total_after(i) = i in 1:64 ? ((UInt64(1) << i) - 1) : throw(DomainError(i)) diff --git a/test/fixtures/grains/results.json b/test/fixtures/grains/results.json new file mode 100644 index 0000000..678a785 --- /dev/null +++ b/test/fixtures/grains/results.json @@ -0,0 +1,31 @@ +{ + "status": "pass", + "version": 2, + "tests": [ + { + "name": "beginning squares", + "test_code": "@test on_square(1) == 1\n@test on_square(2) == 2\n@test on_square(3) == 4\n@test on_square(4) == 8\n@test on_square(16) == 32768\n@test on_square(32) == 2147483648\n@test total_after(1) == 1\n@test total_after(3) == on_square(1) + on_square(2) + on_square(3)", + "status": "pass" + }, + { + "name": "ending squares", + "test_code": "@test total_after(32) < total_after(64)\n@test on_square(64) == 9223372036854775808\n@test total_after(64) == 18446744073709551615", + "status": "pass" + }, + { + "name": "Invalid values » Zero", + "test_code": "@test on_square(0)\n@test total_after(0)", + "status": "pass" + }, + { + "name": "Invalid values » Negative", + "test_code": "@test on_square(-1)\n@test total_after(-1)", + "status": "pass" + }, + { + "name": "Invalid values » Greater than 64", + "test_code": "@test on_square(65)\n@test total_after(65)", + "status": "pass" + } + ] +} diff --git a/test/fixtures/grains/runtests.jl b/test/fixtures/grains/runtests.jl new file mode 100644 index 0000000..b458fda --- /dev/null +++ b/test/fixtures/grains/runtests.jl @@ -0,0 +1,37 @@ +using Test + +include("grains.jl") + +@testset "beginning squares" begin + @test on_square(1) == 1 + @test on_square(2) == 2 + @test on_square(3) == 4 + @test on_square(4) == 8 + @test on_square(16) == 32768 + @test on_square(32) == 2147483648 + @test total_after(1) == 1 + @test total_after(3) == on_square(1) + on_square(2) + on_square(3) +end + +@testset "ending squares" begin + @test total_after(32) < total_after(64) + @test on_square(64) == 9223372036854775808 + @test total_after(64) == 18446744073709551615 +end + +@testset "Invalid values" begin + @testset "Zero" begin + @test_throws DomainError on_square(0) + @test_throws DomainError total_after(0) + end + + @testset "Negative" begin + @test_throws DomainError on_square(-1) + @test_throws DomainError total_after(-1) + end + + @testset "Greater than 64" begin + @test_throws DomainError on_square(65) + @test_throws DomainError total_after(65) + end +end diff --git a/test/fixtures/nested/results.json b/test/fixtures/nested/results.json index 8d8e7a5..b5bba6e 100644 --- a/test/fixtures/nested/results.json +++ b/test/fixtures/nested/results.json @@ -4,6 +4,7 @@ "tests": [ { "name": "outer » inner 1.1", + "test_code": "@test 1 == 1", "status": "pass" }, { @@ -26,16 +27,18 @@ }, { "name": "outer 2", + "test_code": "@test 1 == 1", "status": "pass" }, { "name": "outer 3.1", "test_code": "@test error(\"\")", "status": "error", - "message": "\nStacktrace:\n [1] error(s::String)\n @ Base ./error.jl:33\n [2] macro expansion\n @ ./runtests.jl:28 [inlined]\n [3] macro expansion\n @ [...]/julia/stdlib/v1.7/Test/src/Test.jl:1283 [inlined]\n [4] top-level scope\n @ ./runtests.jl:28" + "message": "" }, { "name": "outer 3.2", + "test_code": "@test true", "status": "pass" }, { diff --git a/test/fixtures/one_fail/results.json b/test/fixtures/one_fail/results.json index ef8cb59..1f0ddb1 100644 --- a/test/fixtures/one_fail/results.json +++ b/test/fixtures/one_fail/results.json @@ -4,6 +4,7 @@ "tests": [ { "name": "first test", + "test_code": "@test 1 == 1", "status": "pass" }, { @@ -14,6 +15,7 @@ }, { "name": "third test", + "test_code": "@test 3 == 3", "status": "pass" } ] diff --git a/test/fixtures/one_passing/results.json b/test/fixtures/one_passing/results.json index 02e2e59..e15a942 100644 --- a/test/fixtures/one_passing/results.json +++ b/test/fixtures/one_passing/results.json @@ -4,6 +4,7 @@ "tests": [ { "name": "first test", + "test_code": "@test 1 == 1", "status": "pass" } ] diff --git a/test/fixtures/syntax_error/results.json b/test/fixtures/syntax_error/results.json index 4b5da00..40ab6d8 100644 --- a/test/fixtures/syntax_error/results.json +++ b/test/fixtures/syntax_error/results.json @@ -1,6 +1,6 @@ { "status": "error", - "message": "LoadError: syntax: incomplete: invalid string syntax\nStacktrace:\n [1] top-level scope\n @ ./runtests.jl:5\n [2] include(mod::Module, _path::String)\n @ Base ./Base.jl:386\n [3] include\n @ ~/projects/exercism/repos/julia-test-runner/src/ExercismTestReports.jl:1 [inlined]\n [4] macro expansion\n @ ~/projects/exercism/repos/julia-test-runner/src/runner.jl:14 [inlined]\n [5] macro expansion\n @ [...]/julia/stdlib/v1.7/Test/src/Test.jl:1283 [inlined]\n [6] macro expansion\n @ ~/projects/exercism/repos/julia-test-runner/src/runner.jl:14 [inlined]\n [7] macro expansion\n @ ~/.julia/packages/Suppressor/nTjgZ/src/Suppressor.jl:127 [inlined]\n [8] runtests(testfile::String)\n @ ExercismTestReports ~/projects/exercism/repos/julia-test-runner/src/runner.jl:13\n [9] test_runner(exercise_slug::String, solution_dir::String)\n @ ExercismTestReports ~/projects/exercism/repos/julia-test-runner/src/ExercismTestReports.jl:18\n [10] macro expansion\n @ ~/projects/exercism/repos/julia-test-runner/test/runtests.jl:22 [inlined]\n [11] top-level scope\n @ [...]/julia/stdlib/v1.7/Test/src/Test.jl:1226\n [12] include(fname::String)\n @ Base.MainInclude ./client.jl:444\n [13] top-level scope\n @ none:6\n [14] eval\n @ ./boot.jl:360 [inlined]\n [15] exec_options(opts::Base.JLOptions)\n @ Base ./client.jl:261\n [16] _start()\n @ Base ./client.jl:485\nin expression starting at ./runtests.jl:5", + "message": "LoadError: syntax: incomplete: invalid string syntax", "version": 2, "tests": [] } diff --git a/test/fixtures/test_logs/results.json b/test/fixtures/test_logs/results.json index 3a10ba8..b6c6ecc 100644 --- a/test/fixtures/test_logs/results.json +++ b/test/fixtures/test_logs/results.json @@ -3,7 +3,7 @@ "version": 2, "tests": [ { - "name": "", + "name": "nothing", "test_code": "@test_logs (:info, \"Blabla\") nothing", "status": "fail", "message": "Log Test Failed at ./runtests.jl:3\n Expression: nothing\n Log Pattern: (:info, \"Blabla\")\n Captured Logs: \n" diff --git a/test/fixtures/truncated_output/results.json b/test/fixtures/truncated_output/results.json index 31e9b3d..e8ef2f0 100644 --- a/test/fixtures/truncated_output/results.json +++ b/test/fixtures/truncated_output/results.json @@ -4,6 +4,7 @@ "tests": [ { "name": "first test", + "test_code": "@test 1 == 1", "status": "pass", "output": "Output was truncated. Please limit to 500 chars\n\nxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" } diff --git a/test/runtests.jl b/test/runtests.jl index c525f04..51f81fb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using ExercismTestReports using Test +using JSON import InteractiveUtils # When running tests on this project, the solution_dir path is likely to be @@ -18,6 +19,13 @@ const TMP_FIXTURES = joinpath(TEMPDIR, "fixtures") ]) end +# Shorten the error messages so we don't need to care about specific paths, file numbers, etc. +function shorten_messages!(result) + if result["status"] == "error" && haskey(result, "message") + result["message"] = split(result["message"], '\n')[1] + end +end + @testset for fixture in readdir(TMP_FIXTURES) results = test_runner("", "$TMP_FIXTURES/$fixture/") if endswith(fixture, "syntax_error") @@ -26,6 +34,12 @@ end TEST_PREFIX = "~/projects/exercism/repos/julia-test-runner" results = replace(results, HOME_PREFIX => TEST_PREFIX) end + + dict_results = JSON.parse(results) + shorten_messages!(dict_results) # For syntax errors + map(shorten_messages!, dict_results["tests"]) + results = JSON.json(dict_results, 4) + reference = "$FIXTURES/$fixture/results.json" if isfile(reference) ref = read(reference, String)