Skip to content

Commit bd9d9d7

Browse files
maleadtmaleadt
committed
Rework args to support running filtered tests.
1 parent a13c7e0 commit bd9d9d7

File tree

2 files changed

+145
-91
lines changed

2 files changed

+145
-91
lines changed

‎README.md‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,18 +58,23 @@ testsuite = Dict(
5858
runtests(MyModule, ARGS; testsuite)
5959
```
6060

61-
You can also use `find_tests` to automatically discover tests and then filter or modify them:
61+
You can also use `find_tests` to automatically discover tests and then filter or modify them. This requires manually parsing arguments so that filtering is only applied when the user did not request specific tests to run:
6262

6363
```julia
6464
# Start with autodiscovered tests
6565
testsuite = find_tests(pwd())
6666

67-
# Remove tests that shouldn't run on Windows
68-
if Sys.iswindows()
69-
delete!(testsuite, "ext/specialfunctions")
67+
# Parse arguments
68+
args = parse_args(ARGS)
69+
70+
if filter_tests!(testsuite, args)
71+
# Remove tests that shouldn't run on Windows
72+
if Sys.iswindows()
73+
delete!(testsuite, "ext/specialfunctions")
74+
end
7075
end
7176

72-
runtests(MyModule, ARGS; testsuite)
77+
runtests(MyModule, args; testsuite)
7378
```
7479

7580
### Provide defaults

‎src/ParallelTestRunner.jl‎

Lines changed: 135 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module ParallelTestRunner
22

3-
export runtests, addworkers, addworker, find_tests
3+
export runtests, addworkers, addworker, find_tests, parse_args, filter_tests!
44

55
using Malt
66
using Dates
@@ -35,28 +35,6 @@ end
3535

3636
const max_worker_rss = JULIA_TEST_MAXRSS_MB * 2^20
3737

38-
# parse some command-line arguments
39-
function extract_flag!(args, flag, default = nothing; typ = typeof(default))
40-
for f in args
41-
if startswith(f, flag)
42-
# Check if it's just `--flag` or if it's `--flag=foo`
43-
if f != flag
44-
val = split(f, '=')[2]
45-
if !(typ === Nothing || typ <: AbstractString)
46-
val = parse(typ, val)
47-
end
48-
else
49-
val = default
50-
end
51-
52-
# Drop this value from our args
53-
filter!(x -> x != f, args)
54-
return (true, val)
55-
end
56-
end
57-
return (false, default)
58-
end
59-
6038
function with_testset(f, testset)
6139
@static if VERSION >= v"1.13.0-DEV.1044"
6240
Test.@with_testset testset f()
@@ -487,21 +465,121 @@ function find_tests(dir::String)
487465
return tests
488466
end
489467

468+
struct ParsedArgs
469+
jobs::Union{Some{Int}, Nothing}
470+
verbose::Union{Some{Nothing}, Nothing}
471+
quickfail::Union{Some{Nothing}, Nothing}
472+
list::Union{Some{Nothing}, Nothing}
473+
474+
custom::Dict{String,Any}
475+
476+
positionals::Vector{String}
477+
end
478+
479+
# parse some command-line arguments
480+
function extract_flag!(args, flag; typ = Nothing)
481+
for f in args
482+
if startswith(f, flag)
483+
# Check if it's just `--flag` or if it's `--flag=foo`
484+
val = if f == flag
485+
nothing
486+
else
487+
parts = split(f, '=')
488+
if typ === Nothing || typ <: AbstractString
489+
parts[2]
490+
else
491+
parse(typ, parts[2])
492+
end
493+
end
494+
495+
# Drop this value from our args
496+
filter!(x -> x != f, args)
497+
return Some(val)
498+
end
499+
end
500+
return nothing
501+
end
502+
503+
function parse_args(args; custom::Array{String} = String[])
504+
args = copy(args)
505+
506+
help = extract_flag!(args, "--help")
507+
if help !== nothing
508+
println(
509+
"""
510+
Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...]
511+
512+
--help Show this text.
513+
--list List all available tests.
514+
--verbose Print more information during testing.
515+
--quickfail Fail the entire run as soon as a single test errored.
516+
--jobs=N Launch `N` processes to perform tests.
517+
518+
Remaining arguments filter the tests that will be executed."""
519+
)
520+
exit(0)
521+
end
522+
523+
jobs = extract_flag!(args, "--jobs"; typ = Int)
524+
verbose = extract_flag!(args, "--verbose")
525+
quickfail = extract_flag!(args, "--quickfail")
526+
list = extract_flag!(args, "--list")
527+
528+
custom_args = Dict{String,Any}()
529+
for flag in custom
530+
custom_args[flag] = extract_flag!(args, "--$flag")
531+
end
532+
533+
## no options should remain
534+
optlike_args = filter(startswith("-"), args)
535+
if !isempty(optlike_args)
536+
error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)")
537+
end
538+
539+
return ParsedArgs(jobs, verbose, quickfail, list, custom_args, args)
540+
end
541+
542+
"""
543+
filter_tests!(testsuite, args::ParsedArgs) -> Bool
544+
545+
Filter tests in `testsuite` based on command-line arguments in `args`.
546+
547+
Returns `true` if additional filtering may be done by the caller, `false` otherwise.
548+
"""
549+
function filter_tests!(testsuite, args::ParsedArgs)
550+
# the user did not request specific tests, so let the caller do its own filtering
551+
isempty(args.positionals) && return true
552+
553+
# only select tests matching positional arguments
554+
tests = collect(keys(testsuite))
555+
for test in tests
556+
if !any(arg -> startswith(test, arg), args.positionals)
557+
delete!(testsuite, test)
558+
end
559+
end
560+
561+
# the user requested specific tests, so don't allow further filtering
562+
return false
563+
end
564+
490565
"""
491-
runtests(mod::Module, ARGS; testsuite::Dict{String,Expr}=find_tests(pwd()),
492-
RecordType = TestRecord,
493-
init_code = :(),
494-
test_worker = Returns(nothing),
495-
stdout = Base.stdout,
496-
stderr = Base.stderr)
566+
runtests(mod::Module, args::ParsedArgs;
567+
testsuite::Dict{String,Expr}=find_tests(pwd()),
568+
RecordType = TestRecord,
569+
init_code = :(),
570+
test_worker = Returns(nothing),
571+
stdout = Base.stdout,
572+
stderr = Base.stderr)
573+
runtests(mod::Module, ARGS; ...)
497574
498575
Run Julia tests in parallel across multiple worker processes.
499576
500577
## Arguments
501578
502579
- `mod`: The module calling runtests
503580
- `ARGS`: Command line arguments array, typically from `Base.ARGS`. When you run the tests
504-
with `Pkg.test`, this can be changed with the `test_args` keyword argument.
581+
with `Pkg.test`, this can be changed with the `test_args` keyword argument. If the caller
582+
needs to accept args too, consider using `parse_args` to parse the arguments first.
505583
506584
Several keyword arguments are also supported:
507585
@@ -542,87 +620,57 @@ runtests(MyModule, ARGS)
542620
# Run only tests matching "integration"
543621
runtests(MyModule, ["integration"])
544622
545-
# Customize the test suite
546-
testsuite = find_tests(pwd())
547-
delete!(testsuite, "slow_test") # Remove a specific test
548-
runtests(MyModule, ARGS; testsuite)
549-
550-
# Define a custom test suite manually
623+
# Define a custom test suite
551624
testsuite = Dict(
552625
"custom" => quote
553626
@test 1 + 1 == 2
554627
end
555628
)
556629
runtests(MyModule, ARGS; testsuite)
557630
558-
# Use custom test record type
559-
runtests(MyModule, ARGS; RecordType = MyCustomTestRecord)
631+
# Customize the test suite
632+
testsuite = find_tests(pwd())
633+
args = parse_args(ARGS)
634+
if filter_tests!(testsuite, args)
635+
# Remove a specific test
636+
delete!(testsuite, "slow_test")
637+
end
638+
runtests(MyModule, args; testsuite)
560639
```
561640
562641
## Memory Management
563642
564643
Workers are automatically recycled when they exceed memory limits to prevent out-of-memory
565644
issues during long test runs. The memory limit is set based on system architecture.
566645
"""
567-
function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(pwd()),
646+
function runtests(mod::Module, args::ParsedArgs;
647+
testsuite::Dict{String,Expr} = find_tests(pwd()),
568648
RecordType = TestRecord, init_code = :(), test_worker = Returns(nothing),
569649
stdout = Base.stdout, stderr = Base.stderr)
570650
#
571651
# set-up
572652
#
573653

574-
do_help, _ = extract_flag!(ARGS, "--help")
575-
if do_help
576-
println(
577-
"""
578-
Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...]
579-
580-
--help Show this text.
581-
--list List all available tests.
582-
--verbose Print more information during testing.
583-
--quickfail Fail the entire run as soon as a single test errored.
584-
--jobs=N Launch `N` processes to perform tests.
585-
586-
Remaining arguments filter the tests that will be executed."""
587-
)
654+
# list tests, if requested
655+
if args.list !== nothing
656+
println(stdout, "Available tests:")
657+
for test in keys(testsuite)
658+
println(stdout, " - $test")
659+
end
588660
exit(0)
589661
end
590-
set_jobs, jobs = extract_flag!(ARGS, "--jobs"; typ = Int)
591-
do_verbose, _ = extract_flag!(ARGS, "--verbose")
592-
do_quickfail, _ = extract_flag!(ARGS, "--quickfail")
593-
do_list, _ = extract_flag!(ARGS, "--list")
594-
## no options should remain
595-
optlike_args = filter(startswith("-"), ARGS)
596-
if !isempty(optlike_args)
597-
error("Unknown test options `$(join(optlike_args, " "))` (try `--help` for usage instructions)")
598-
end
662+
663+
# filter tests
664+
filter_tests!(testsuite, args)
599665

600666
# determine test order
601667
tests = collect(keys(testsuite))
602668
Random.shuffle!(tests)
603669
historical_durations = load_test_history(mod)
604670
sort!(tests, by = x -> -get(historical_durations, x, Inf))
605671

606-
# list tests, if requested
607-
if do_list
608-
println(stdout, "Available tests:")
609-
for test in sort(tests)
610-
println(stdout, " - $test")
611-
end
612-
exit(0)
613-
end
614-
615-
# filter tests based on command-line arguments
616-
if !isempty(ARGS)
617-
filter!(tests) do test
618-
any(arg -> startswith(test, arg), ARGS)
619-
end
620-
end
621-
622672
# determine parallelism
623-
if !set_jobs
624-
jobs = default_njobs()
625-
end
673+
jobs = something(args.jobs, default_njobs())
626674
jobs = clamp(jobs, 1, length(tests))
627675
println(stdout, "Running $jobs tests in parallel. If this is too many, specify the `--jobs=N` argument to the tests, or set the `JULIA_CPU_THREADS` environment variable.")
628676
workers = addworkers(min(jobs, length(tests)))
@@ -761,7 +809,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
761809
test_name, wrkr = msg[2], msg[3]
762810

763811
# Optionally print verbose started message
764-
if do_verbose
812+
if args.verbose !== nothing
765813
clear_status()
766814
print_test_started(RecordType, wrkr, test_name, io_ctx)
767815
end
@@ -868,7 +916,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
868916
# One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException
869917
@assert result isa Exception
870918
put!(printer_channel, (:crashed, test, worker_id(wrkr)))
871-
if do_quickfail
919+
if args.quickfail !== nothing
872920
stop_work()
873921
end
874922

@@ -977,7 +1025,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
9771025
return testset
9781026
end
9791027
t1 = time()
980-
o_ts = create_testset("Overall"; start=t0, stop=t1, verbose=do_verbose)
1028+
o_ts = create_testset("Overall"; start=t0, stop=t1, verbose=!isnothing(args.verbose))
9811029
function collect_results()
9821030
with_testset(o_ts) do
9831031
completed_tests = Set{String}()
@@ -1054,6 +1102,7 @@ function runtests(mod::Module, ARGS; testsuite::Dict{String,Expr} = find_tests(p
10541102
end
10551103

10561104
return
1057-
end # runtests
1105+
end
1106+
runtests(mod::Module, ARGS; kwargs...) = runtests(mod, parse_args(ARGS); kwargs...)
10581107

1059-
end # module ParallelTestRunner
1108+
end

0 commit comments

Comments
 (0)