Start tests in sorted order also with multiple threads

.github/workflows/UnitTests.yml

@@ -60,3 +60,5 @@ jobs:
           arch: ${{ matrix.julia-arch }}
       - uses: julia-actions/cache@v3
       - uses: julia-actions/julia-runtest@v1
+        env:
+          JULIA_NUM_THREADS: "auto"

src/ParallelTestRunner.jl

@@ -1189,10 +1189,12 @@ function _runtests(mod::Module, args::ParsedArgs;
     #
 
     tests_to_start = Threads.Atomic{Int}(length(tests))
+    next_test = Threads.Atomic{Int}(1)
     try
-        @sync for test in tests
+        @sync for _ in 1:length(tests)
             push!(worker_tasks, Threads.@spawn begin
                 local p = nothing
+                local test
                 acquired = false
                 try
                     Base.acquire(sem)
@@ -1202,6 +1204,11 @@ function _runtests(mod::Module, args::ParsedArgs;
 
                     done && return
 
+                    # with multiple threads, tasks reach this point in arbitrary order,
+                    # so pick the next test to run only now, rather than at spawn time,
+                    # to preserve the sorted test order (issue #139)
+                    test = tests[Threads.atomic_add!(next_test, 1)]
+
                     test_t0 = @lock running_tests begin
                         test_t0 = time()
                         running_tests[][test] = test_t0

test/runtests.jl

@@ -71,6 +71,36 @@ end
     @test contains(str, "SUCCESS")
 end
 
+@testset "test start order" begin
+    # Tests must start in the given order, regardless of the
+    # number of threads of the host Julia process (issue #139).
+    testsuite = Dict(
+        "sort$(n)" => :(@test true)
+        for n in 1:5
+    )
+    tests = ["sort$(n)" for n in 1:length(testsuite)]
+
+    io = IOBuffer()
+    # with a single job, tests start strictly in acquisition order
+    ParallelTestRunner._runtests(
+        ParallelTestRunner, parse_args(["--jobs=1", "--verbose"]);
+        testsuite,
+        tests,
+        historical_durations=Dict{String, Float64}(),
+        stdout=io,
+        stderr=io,
+    )
+
+    str = String(take!(io))
+    @test contains(str, "SUCCESS")
+    started_at = map(tests) do name
+        m = match(Regex("$(name) .+ started at"), str)
+        @test m !== nothing
+        m === nothing ? typemax(Int) : m.offset
+    end
+    @test issorted(started_at)
+end
+
 @testset "init code" begin
     init_code = quote
         using Test