From 009e2af77f09afe1db437389d8128a5a4d8b0cea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Thu, 2 Jul 2026 22:25:18 +0100 Subject: [PATCH 1/3] Start tests in sorted order also with multiple threads Since #119, each test is bound to its own task at spawn time, and all tasks race to acquire the semaphore. With a single thread tasks happen to reach the semaphore in spawn order, but with multiple threads the acquisition order is arbitrary, defeating the descending-historical- duration sort (#139). Instead, hand out tests in sorted order at acquisition time, via an atomic fetch-add index claimed only once a task holds a semaphore slot and a worker. Fixes #139. Co-Authored-By: Claude Fable 5 --- src/ParallelTestRunner.jl | 9 ++++++++- test/runtests.jl | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index b08642d..35f332b 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -1189,10 +1189,12 @@ function _runtests(mod::Module, args::ParsedArgs; # tests_to_start = Threads.Atomic{Int}(length(tests)) + next_test = Threads.Atomic{Int}(1) try - @sync for test in tests + @sync for _ in 1:length(tests) push!(worker_tasks, Threads.@spawn begin local p = nothing + local test acquired = false try Base.acquire(sem) @@ -1202,6 +1204,11 @@ function _runtests(mod::Module, args::ParsedArgs; done && return + # with multiple threads, tasks reach this point in arbitrary order, + # so pick the next test to run only now, rather than at spawn time, + # to preserve the sorted test order (issue #139) + test = tests[Threads.atomic_add!(next_test, 1)] + test_t0 = @lock running_tests begin test_t0 = time() running_tests[][test] = test_t0 diff --git a/test/runtests.jl b/test/runtests.jl index 1526681..c42ac6f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,6 +5,10 @@ cd(@__DIR__) include(joinpath(@__DIR__, "utils.jl")) +# dummy module to give the "test start order" testset its own history file, +# without clobbering the history of `ParallelTestRunner` used by other testsets +module SortingHistoryDummy end + @testset "ParallelTestRunner" verbose=true begin @testset "basic use" begin @@ -71,6 +75,34 @@ end @test contains(str, "SUCCESS") end +@testset "test start order" begin + # tests must start in descending order of historical duration, regardless of + # the number of threads of this process (issue #139) + testsuite = Dict( + name => :(@test true) + for name in ("sort1", "sort2", "sort3", "sort4", "sort5") + ) + # deliberately not in alphabetical order, so that tests accidentally started in + # alphabetical (or insertion) order would not pass the test below + expected_order = ["sort3", "sort1", "sort5", "sort2", "sort4"] + ParallelTestRunner.save_test_history(SortingHistoryDummy, + Dict(name => Float64(length(expected_order) - idx) + for (idx, name) in enumerate(expected_order))) + + io = IOBuffer() + # with a single job, tests start strictly in acquisition order + runtests(SortingHistoryDummy, ["--jobs=1", "--verbose"]; testsuite, stdout=io, stderr=io) + + str = String(take!(io)) + @test contains(str, "SUCCESS") + started_at = map(expected_order) do name + m = match(Regex("$(name) .+ started at"), str) + @test m !== nothing + m === nothing ? typemax(Int) : m.offset + end + @test issorted(started_at) +end + @testset "init code" begin init_code = quote using Test From 0d937b663d238c89fef1b1442d59c5e66b368a82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Fri, 3 Jul 2026 00:10:33 +0100 Subject: [PATCH 2/3] [CI] Run tests with multiple julia threads --- .github/workflows/UnitTests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/UnitTests.yml b/.github/workflows/UnitTests.yml index b747dc9..28d131c 100644 --- a/.github/workflows/UnitTests.yml +++ b/.github/workflows/UnitTests.yml @@ -60,3 +60,5 @@ jobs: arch: ${{ matrix.julia-arch }} - uses: julia-actions/cache@v3 - uses: julia-actions/julia-runtest@v1 + env: + JULIA_NUM_THREADS: "auto" From 87db456be9120222ca9e85cf3ff7e05228503341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:52:24 +0100 Subject: [PATCH 3/3] Avoid using dummy module for testing launch order --- test/runtests.jl | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index c42ac6f..888ecb0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,10 +5,6 @@ cd(@__DIR__) include(joinpath(@__DIR__, "utils.jl")) -# dummy module to give the "test start order" testset its own history file, -# without clobbering the history of `ParallelTestRunner` used by other testsets -module SortingHistoryDummy end - @testset "ParallelTestRunner" verbose=true begin @testset "basic use" begin @@ -76,26 +72,28 @@ end end @testset "test start order" begin - # tests must start in descending order of historical duration, regardless of - # the number of threads of this process (issue #139) + # Tests must start in the given order, regardless of the + # number of threads of the host Julia process (issue #139). testsuite = Dict( - name => :(@test true) - for name in ("sort1", "sort2", "sort3", "sort4", "sort5") + "sort$(n)" => :(@test true) + for n in 1:5 ) - # deliberately not in alphabetical order, so that tests accidentally started in - # alphabetical (or insertion) order would not pass the test below - expected_order = ["sort3", "sort1", "sort5", "sort2", "sort4"] - ParallelTestRunner.save_test_history(SortingHistoryDummy, - Dict(name => Float64(length(expected_order) - idx) - for (idx, name) in enumerate(expected_order))) + tests = ["sort$(n)" for n in 1:length(testsuite)] io = IOBuffer() # with a single job, tests start strictly in acquisition order - runtests(SortingHistoryDummy, ["--jobs=1", "--verbose"]; testsuite, stdout=io, stderr=io) + ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--jobs=1", "--verbose"]); + testsuite, + tests, + historical_durations=Dict{String, Float64}(), + stdout=io, + stderr=io, + ) str = String(take!(io)) @test contains(str, "SUCCESS") - started_at = map(expected_order) do name + started_at = map(tests) do name m = match(Regex("$(name) .+ started at"), str) @test m !== nothing m === nothing ? typemax(Int) : m.offset