diff --git a/base/interactiveutil.jl b/base/interactiveutil.jl index 2457bb54023a0..66c880e478dae 100644 --- a/base/interactiveutil.jl +++ b/base/interactiveutil.jl @@ -697,19 +697,23 @@ end """ Base.runtests(tests=["all"], numcores=ceil(Int, Sys.CPU_CORES / 2); - exit_on_error=false) + exit_on_error=false, [seed]) Run the Julia unit tests listed in `tests`, which can be either a string or an array of strings, using `numcores` processors. If `exit_on_error` is `false`, when one test fails, all remaining tests in other files will still be run; they are otherwise discarded, when `exit_on_error == true`. +If a seed is provided via the keyword argument, it is used to seed the +global RNG in the context where the tests are run; otherwise the seed is chosen randomly. """ function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2); - exit_on_error=false) + exit_on_error=false, + seed::Union{BitInteger,Void}=nothing) if isa(tests,AbstractString) tests = split(tests) end exit_on_error && push!(tests, "--exit-on-error") + seed != nothing && push!(tests, "--seed=0x$(hex(seed % UInt128))") # cast to UInt128 to avoid a minus sign ENV2 = copy(ENV) ENV2["JULIA_CPU_CORES"] = "$numcores" try diff --git a/test/choosetests.jl b/test/choosetests.jl index 2d82a08631d0d..5320e104c925e 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -2,7 +2,7 @@ @doc """ -`tests, net_on, exit_on_error = choosetests(choices)` selects a set of tests to be +`tests, net_on, exit_on_error, seed = choosetests(choices)` selects a set of tests to be run. `choices` should be a vector of test names; if empty or set to `["all"]`, all tests are selected. @@ -10,14 +10,21 @@ This function also supports "test collections": specifically, "linalg" refers to collections of tests in the correspondingly-named directories. -Upon return, `tests` is a vector of fully-expanded test names, -`net_on` is true if networking is available (required for some tests), -and `exit_on_error` is true if an error in one test should cancel -remaining tests to be run (otherwise, all tests are run unconditionally). - -Two options can be passed to `choosetests` by including a special token -in the `choices` argument: "--skip", which makes all tests coming after -be skipped, and "--exit-on-error" which sets the value of `exit_on_error`. +Upon return: + - `tests` is a vector of fully-expanded test names, + - `net_on` is true if networking is available (required for some tests), + - `exit_on_error` is true if an error in one test should cancel + remaining tests to be run (otherwise, all tests are run unconditionally), + - `seed` is a seed which will be used to initialize the global RNG for each + test to be run. + +Three options can be passed to `choosetests` by including a special token +in the `choices` argument: + - "--skip", which makes all tests coming after be skipped, + - "--exit-on-error" which sets the value of `exit_on_error`, + - "--seed=SEED", which sets the value of `seed` to `SEED` + (parsed as an `UInt128`); `seed` is otherwise initialized randomly. + This option can be used to reproduce failed tests. """ -> function choosetests(choices = []) testnames = [ @@ -58,6 +65,7 @@ function choosetests(choices = []) tests = [] skip_tests = [] exit_on_error = false + seed = rand(RandomDevice(), UInt128) for (i, t) in enumerate(choices) if t == "--skip" @@ -65,6 +73,8 @@ function choosetests(choices = []) break elseif t == "--exit-on-error" exit_on_error = true + elseif startswith(t, "--seed=") + seed = parse(UInt128, t[8:end]) else push!(tests, t) end @@ -177,5 +187,5 @@ function choosetests(choices = []) filter!(x -> !(x in skip_tests), tests) - tests, net_on, exit_on_error + tests, net_on, exit_on_error, seed end diff --git a/test/runtests.jl b/test/runtests.jl index 7a49b8697096d..1bfc2eaced2f5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,7 +4,7 @@ using Test include("choosetests.jl") include("testenv.jl") -tests, net_on, exit_on_error = choosetests(ARGS) +tests, net_on, exit_on_error, seed = choosetests(ARGS) tests = unique(tests) const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB") @@ -55,7 +55,7 @@ cd(dirname(@__FILE__)) do local resp wrkr = p try - resp = remotecall_fetch(runtests, wrkr, test) + resp = remotecall_fetch(runtests, wrkr, test; seed=seed) catch e resp = [e] end @@ -105,7 +105,7 @@ cd(dirname(@__FILE__)) do n > 1 && print("\tFrom worker 1:\t") local resp try - resp = eval(Expr(:call, () -> runtests(t))) # runtests is defined by the include above + resp = eval(Expr(:call, () -> runtests(t, seed=seed))) # runtests is defined by the include above catch e resp = [e] end @@ -188,7 +188,8 @@ cd(dirname(@__FILE__)) do else println(" \033[31;1mFAILURE\033[0m\n") skipped > 0 && - println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.\n") + println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.") + println("The global RNG seed was 0x$(hex(seed)).\n") Test.print_test_errors(o_ts) throw(Test.FallbackTestSetException("Test run finished with errors")) end diff --git a/test/testdefs.jl b/test/testdefs.jl index 02b2f2c3cc10d..74dca4f76b880 100644 --- a/test/testdefs.jl +++ b/test/testdefs.jl @@ -2,7 +2,7 @@ using Test -function runtests(name, isolate=true) +function runtests(name, isolate=true; seed=nothing) old_print_setting = Test.TESTSET_PRINT_ENABLE[] Test.TESTSET_PRINT_ENABLE[] = false try @@ -17,6 +17,7 @@ function runtests(name, isolate=true) @eval(m, using Test) ex = quote @timed @testset $"$name" begin + srand($seed) include($"$name.jl") end end