From e17cf5d94582ea2bebb493e83d0e09951ad86d79 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Tue, 21 Jul 2026 10:47:27 -0500 Subject: [PATCH 1/4] Add recycle_on_failure and retries options to runtests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test that corrupts process-wide state — the motivating case is a GPU whose driver ends up in a state where every subsequent allocation in the process fails — poisons every later test scheduled onto the same worker, turning one bad test into a cascade of failed files. The Distributed- based harness this package was extracted from recycled a worker after any failed test; restore that behavior behind `recycle_on_failure = true`, alongside the existing max-rss and crash recycling. With `retries = N`, tests that did not pass are re-run up to N times after the main run completes: sequentially, on a single fresh worker, with all other workers stopped. Parallel test runs create resource contention (several workers sharing one GPU or a limited amount of RAM), so a failure can mean "lost the resource race" rather than "broken": re-running on an otherwise-idle system distinguishes the two. Tests that failed due to contention reliably pass on the idle retry, while deterministic failures fail again and are reported exactly once — only the final attempt of each test enters the results, and retried tests are visibly marked in the output. Both options default to off. --- src/ParallelTestRunner.jl | 73 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/src/ParallelTestRunner.jl b/src/ParallelTestRunner.jl index 2704bf3..6c6d512 100644 --- a/src/ParallelTestRunner.jl +++ b/src/ParallelTestRunner.jl @@ -950,6 +950,17 @@ runtests(MyPackage, ARGS; serial=["big_alloc_test", "huge_matrix"]) Workers are automatically recycled when they exceed memory limits to prevent out-of-memory issues during long test runs. The memory limit is set based on system architecture. + +## Failure Handling + +With `recycle_on_failure = true`, a worker is recycled after any test that did not pass, so +a test that corrupts process-wide state (e.g. wedges a GPU driver) cannot poison subsequent +tests on the same worker. + +With `retries = N` (default 0), tests that did not pass are re-run up to `N` times after +the main run completes — sequentially, on a single fresh worker, with all other workers +stopped — so tests that failed due to resource pressure from concurrent workers get an +otherwise-idle system. Only the final attempt of each test is reported. """ function runtests(mod::Module, args::ParsedArgs; testsuite::Dict{String,Expr} = find_tests(pwd()), @@ -964,6 +975,8 @@ function runtests(mod::Module, args::ParsedArgs; stdout = Base.stdout, stderr = Base.stderr, max_worker_rss = get_max_worker_rss(), + recycle_on_failure::Bool = false, + retries::Integer = 0, ) # # set-up @@ -1012,6 +1025,8 @@ function runtests(mod::Module, args::ParsedArgs; stdout, stderr, max_worker_rss, + recycle_on_failure, + retries, ) end @@ -1033,6 +1048,8 @@ function _runtests(mod::Module, args::ParsedArgs; stdout = Base.stdout, stderr = Base.stderr, max_worker_rss = get_max_worker_rss(), + recycle_on_failure::Bool = false, + retries::Integer = 0, ) # partition into serial and parallel groups @@ -1262,6 +1279,7 @@ function _runtests(mod::Module, args::ParsedArgs; # tests_to_start = Threads.Atomic{Int}(length(tests)) + interrupted = false # After parallel-before-serial: stop extra workers so only one process is alive for # serial tests, but keep one parallel worker so we do not add a third addworker (ID_COUNTER). function drain_pool_leaving_one_worker!(pool, njobs) @@ -1362,6 +1380,11 @@ function _runtests(mod::Module, args::ParsedArgs; # the worker has reached the max-rss limit, recycle it # so future tests start with a smaller working set Malt.stop(wrkr) + elseif recycle_on_failure && anynonpass(result[]) + # a failing test may have left the worker in a bad state + # (e.g. a wedged GPU driver whose every later allocation + # fails); recycle it so future tests get a fresh process + Malt.stop(wrkr) end else # One of Malt.TerminatedWorkerException, Malt.RemoteException, or ErrorException @@ -1418,6 +1441,7 @@ function _runtests(mod::Module, args::ParsedArgs; end end catch err + interrupted = true if !(err isa InterruptException) println(io_ctx.stderr, "\nCaught an error, stopping...") end @@ -1455,6 +1479,55 @@ function _runtests(mod::Module, args::ParsedArgs; end end + # retry failed tests, if requested: sequentially, on a single fresh worker, with every + # other worker gone — tests that failed due to resource pressure (e.g. GPU memory + # oversubscription from concurrent workers) reliably pass on an otherwise-idle system. + # only the retried result is reported; persistent failures fail again and are reported + # exactly once. + if retries > 0 && !interrupted && args.quickfail === nothing + local retry_wrkr = nothing + for round in 1:retries + retryable = [r.test for r in results.value + if r.result isa Exception || anynonpass(r.result[])] + isempty(retryable) && break + println(io_ctx.stdout) + printstyled(io_ctx.stdout, + "Retrying $(length(retryable)) failed test(s) on a fresh worker...\n"; + color = :yellow) + for test in retryable + if retry_wrkr === nothing || !Malt.isrunning(retry_wrkr) + retry_wrkr = addworker(; init_worker_code, io_ctx.color, exename, + exeflags, env) + end + test_t0 = time() + result = try + Malt.remote_eval_wait(Main, retry_wrkr.w, :(import ParallelTestRunner)) + Malt.remote_call_fetch(invokelatest, retry_wrkr.w, runtest, + RecordType, testsuite[test], test, + init_code, test_t0, custom_args) + catch ex + isa(ex, InterruptException) && rethrow() + ex + end + test_t1 = time() + output = @lock retry_wrkr.io String(take!(retry_wrkr.io[])) + filter!(r -> r.test != test, results.value) + push!(results.value, (; test, result, output, test_t0, test_t1)) + if result isa AbstractTestRecord && !anynonpass(result[]) + printstyled(io_ctx.stdout, " $test passed on retry\n"; color = :green) + else + printstyled(io_ctx.stdout, " $test failed again\n"; color = :red) + # don't let a failure contaminate the next retry + Malt.stop(retry_wrkr) + retry_wrkr = nothing + end + end + end + if retry_wrkr !== nothing && Malt.isrunning(retry_wrkr) + Malt.stop(retry_wrkr) + end + end + # print the output generated by each testset # (`@sync` above joined all writers, so `results` is quiescent from here on) for (testname, result, output, _start, _stop) in results.value From 5c4117264ce24a3ff0271ac331002982d3de6721 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Tue, 21 Jul 2026 10:47:27 -0500 Subject: [PATCH 2/4] Bump version to 2.7.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 0885e68..abac725 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ParallelTestRunner" uuid = "d3525ed8-44d0-4b2c-a655-542cee43accc" authors = ["Valentin Churavy "] -version = "2.6.5" +version = "2.7.0" [deps] Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" From 0f283617cb48730066420cf810be9a0eea65207c Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Mon, 3 Aug 2026 11:07:56 -0500 Subject: [PATCH 3/4] Document recycle_on_failure and retries Add a "Failure Handling" section to the advanced usage guide covering both options: why worker recycling after a failure is useful (process-wide state corruption cascading onto later tests on the same worker) and what the retry environment guarantees (all other workers stopped, sequential re-run on a fresh worker, only the final attempt reported, retry worker recycled after a repeat failure). Also mention them in the feature list on the front page, and add a best practice warning against using retries to paper over genuinely broken tests. --- docs/src/advanced.md | 51 ++++++++++++++++++++++++++++++++++++++++++++ docs/src/index.md | 9 ++++++++ 2 files changed, 60 insertions(+) diff --git a/docs/src/advanced.md b/docs/src/advanced.md index 2ac2e93..b749b7d 100644 --- a/docs/src/advanced.md +++ b/docs/src/advanced.md @@ -174,6 +174,55 @@ duration, longest first) and their results appear in the same overall summary. If the user filters tests via positional arguments (e.g. `julia test/runtests.jl unit`), any serial test names that were filtered out are silently removed from the serial list. +## Failure Handling + +Both options described in this section are opt-in and default to off. + +### Recycling Workers after a Failure + +Workers are reused across tests, so a test that corrupts process-wide state — a wedged GPU driver whose every subsequent allocation fails, a global left in an inconsistent state, a library put in an unusable configuration — can make every later test scheduled on that same worker fail too. + +Setting `recycle_on_failure=true` stops the worker after any test that did not pass, so the next test gets a fresh process: + +```julia +runtests(MyPackage, ARGS; recycle_on_failure=true) +``` + +This complements the existing recycling of workers exceeding `max_worker_rss` and of workers that crashed outright. +The cost is worker start-up time (plus re-running `init_worker_code`) after each failure, which is why it is off by default: for a suite whose failures are self-contained it is pure overhead. + +### Retrying Failed Tests + +When several workers compete for a limited resource — GPU memory, RAM, a device that only allows so many contexts — a failure can mean "lost the race for the resource" rather than "the code is broken". +Such a test typically passes when run on its own. + +The `retries` keyword argument re-runs tests that did not pass, up to `N` times, after the main run has completed: + +```julia +runtests(MyPackage, ARGS; retries=1) +``` + +The retry environment is deliberately quiesced: all parallel workers have been stopped by then, and the retried tests run **sequentially on a single fresh worker**, so a test that failed only because of concurrent resource pressure gets an otherwise-idle system. +If a test fails again, its worker is stopped before the next retry, so one failure cannot contaminate the following one. + +Only the final attempt of each test is recorded in the results, so a test that passes on retry is reported as passing and a persistently broken test is reported as failing exactly once. +Retries are visible in the output, so flakiness is surfaced rather than hidden: + +``` +Retrying 2 failed test(s) on a fresh worker... + gpu/memory passed on retry + broken_test failed again +``` + +!!! note + Retries are skipped when the run was interrupted (e.g. `Ctrl+C`) or when `--quickfail` is + in effect, since in both cases the run stopped early on purpose. + +!!! tip + `recycle_on_failure` and `retries` address different halves of the same problem and work + well together: recycling keeps one bad test from cascading onto its worker during the run, + while retries give the tests that did fail a contention-free second chance. + ## Custom Workers For tests that require specific environment variables or Julia flags, you can use the `test_worker` keyword argument to [`runtests`](@ref) to assign tests to custom workers: @@ -303,3 +352,5 @@ function jltest { 1. **Use custom workers sparingly**: Custom workers add overhead. Only use them when tests genuinely require different configurations. 1. **Use `serial` for resource-intensive tests**: If a test allocates significant memory or uses exclusive hardware resources, mark it as serial rather than reducing `--jobs` globally. This keeps the rest of your suite running in parallel. + +1. **Don't paper over real failures with `retries`**: Retries are meant for failures caused by contention between concurrent workers, not for tests that are genuinely broken. Persistent failures still fail after their retries, and retried tests are reported as such, so keep an eye on which tests keep needing a second attempt. diff --git a/docs/src/index.md b/docs/src/index.md index ee69c02..b9dec9d 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -114,6 +114,15 @@ The `serial` keyword argument to [`runtests`](@ref) lets you designate specific for sequential execution, either before or after the parallel batch. See [Serial Tests](@ref) in the advanced usage guide for details. +### Failure Recycling and Retries + +Workers are recycled when they crash or exceed the memory threshold. +Additionally, `recycle_on_failure=true` recycles a worker after any failed test, so a test +that corrupts process-wide state cannot poison later tests, and `retries=N` re-runs failed +tests on an otherwise-idle system, to tell tests broken by resource contention apart from +genuinely broken ones. +See [Failure Handling](@ref) in the advanced usage guide for details. + ### Real-time Progress The test runner provides real-time output showing: From 90f247905ddd84265f30a46ba7a85ace376cf953 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Mon, 3 Aug 2026 11:22:11 -0500 Subject: [PATCH 4/4] Add tests for recycle_on_failure and retries For `recycle_on_failure`, run a fixed sequence of failing and passing tests with a single job and count the workers created: the default reuses one worker for all of them, while `recycle_on_failure=true` needs a fresh worker after each failure. For `retries`, use a test that fails on its first attempt and passes on any subsequent one (recording attempts in a file, since each attempt runs in a different process) to check that a test rescued by a retry is reported as passing, and that it is the only worker alive while it runs. A persistently failing test is checked to exhaust its retries and still be reported exactly once. Also cover that retries are off by default and skipped under `--quickfail`. --- test/runtests.jl | 197 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index 87e29a8..36b190b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1273,6 +1273,203 @@ end end end +@testset "recycle_on_failure" begin + # Call `_runtests` throughout, so that we can enforce a run order, and use a single job, + # so that all tests share the same pool slot: a test only gets a new worker if the + # previous one was recycled. + testsuite = Dict( + "fail1" => :( @test false ), + "pass1" => :( @test true ), + "fail2" => :( @test false ), + "pass2" => :( @test true ), + ) + tests = ["fail1", "pass1", "fail2", "pass2"] + + @testset "workers are reused across failures by default" begin + io = IOBuffer() + old_id_counter = ParallelTestRunner.ID_COUNTER[] + @test_throws Test.FallbackTestSetException begin + ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--jobs=1"]); + testsuite, + tests, + historical_durations=Dict{String, Float64}(), + stdout=io, + stderr=io, + ) + end + str = String(take!(io)) + @test contains(str, "FAILURE") + # A failing test does not recycle its worker, so a single one runs all four tests. + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 1 + end + + @testset "worker is recycled after a failed test" begin + io = IOBuffer() + old_id_counter = ParallelTestRunner.ID_COUNTER[] + @test_throws Test.FallbackTestSetException begin + ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--jobs=1"]); + testsuite, + tests, + historical_durations=Dict{String, Float64}(), + stdout=io, + stderr=io, + recycle_on_failure=true, + ) + end + str = String(take!(io)) + @test contains(str, "FAILURE") + # `fail1` and `fail2` recycle their worker, so `pass1` and `pass2` each need a fresh + # one: 1 initial worker + 2 replacements. + @test ParallelTestRunner.ID_COUNTER[] == old_id_counter + 3 + end +end + +@testset "retries" begin + # A test that fails on its first attempt and passes on any subsequent one, by recording + # attempts in a file: the worker running the retry is a different process, so the marker + # has to live outside of it. + flaky_test(marker, body=:( @test true )) = quote + if isfile($marker) + $body + else + touch($marker) + @test false + end + end + + @testset "failed test passing on retry is reported as passing" begin + mktempdir() do dir + testsuite = Dict( + "flaky" => flaky_test(joinpath(dir, "flaky")), + "passes" => :( @test true ), + ) + io = IOBuffer() + @show_if_error io ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--jobs=1"]); + testsuite, + tests=["flaky", "passes"], + historical_durations=Dict{String, Float64}(), + stdout=io, + stderr=io, + retries=1, + ) + str = String(take!(io)) + # Only the failed test is retried, and its retried result is the one reported. + @test contains(str, "Retrying 1 failed test(s)") + @test contains(str, "flaky passed on retry") + @test !contains(str, "passes passed on retry") + @test contains(str, "SUCCESS") + # Two results in total: the failed attempt of `flaky` was replaced by the + # retried one, rather than reported next to it. + @test contains(str, r"Overall +\| +2 +2 ") + end + end + + @testset "persistent failure is retried and reported once" begin + testsuite = Dict( + "always_fails" => :( @test false ), + "passes" => :( @test true ), + ) + io = IOBuffer() + @test_throws Test.FallbackTestSetException begin + ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--jobs=1"]); + testsuite, + tests=["always_fails", "passes"], + historical_durations=Dict{String, Float64}(), + stdout=io, + stderr=io, + retries=2, + ) + end + str = String(take!(io)) + @test contains(str, "FAILURE") + # Both retry rounds run, and each of them fails again. + @test length(collect(eachmatch(r"always_fails failed again", str))) == 2 + # Despite the three attempts, the test is reported exactly once, as a failure. + @test contains(str, r"always_fails +\| +1 +1 ") + end + + @testset "retried test runs alone" begin + mktempdir() do dir + # On its retry, the flaky test checks it is the only worker left alive. + check_alone = quote + children = _count_child_pids($(getpid())) + if children >= 0 + @test children == 1 + end + end + testsuite = Dict( + "flaky" => flaky_test(joinpath(dir, "flaky"), check_alone), + "pass1" => :( @test true ), + "pass2" => :( @test true ), + "pass3" => :( @test true ), + ) + io = IOBuffer() + @show_if_error io ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--jobs=3"]); + testsuite, + tests=["flaky", "pass1", "pass2", "pass3"], + historical_durations=Dict{String, Float64}(), + init_code=:(include($(joinpath(@__DIR__, "utils.jl")))), + stdout=io, + stderr=io, + retries=1, + ) + str = String(take!(io)) + @test contains(str, "flaky passed on retry") + @test contains(str, "SUCCESS") + end + end + + @testset "no retries by default" begin + mktempdir() do dir + testsuite = Dict("flaky" => flaky_test(joinpath(dir, "flaky"))) + io = IOBuffer() + @test_throws Test.FallbackTestSetException begin + ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--jobs=1"]); + testsuite, + tests=["flaky"], + historical_durations=Dict{String, Float64}(), + stdout=io, + stderr=io, + ) + end + str = String(take!(io)) + @test !contains(str, "Retrying") + @test contains(str, "FAILURE") + end + end + + @testset "quickfail skips retries" begin + mktempdir() do dir + testsuite = Dict( + "flaky" => flaky_test(joinpath(dir, "flaky")), + "passes" => :( @test true ), + ) + io = IOBuffer() + @test_throws Test.FallbackTestSetException begin + ParallelTestRunner._runtests( + ParallelTestRunner, parse_args(["--quickfail", "--jobs=1"]); + testsuite, + tests=["flaky", "passes"], + historical_durations=Dict{String, Float64}(), + stdout=io, + stderr=io, + retries=1, + ) + end + str = String(take!(io)) + # The run stopped early on purpose, retrying would defeat that. + @test !contains(str, "Retrying") + @test contains(str, "FAILURE") + end + end +end + # This testset should always be the last one, don't add anything after this. # We want to make sure there are no running workers at the end of the tests. @testset "no workers running" begin