Merge pull request JuliaLang#16924 from JuliaLang/rf/seed-tests

enable seeding of GLOBAL_RNG when running tests

base/interactiveutil.jl

@@ -697,19 +697,23 @@ end
 
 """
  Base.runtests(tests=["all"], numcores=ceil(Int, Sys.CPU_CORES / 2);
- exit_on_error=false)
+ exit_on_error=false, [seed])
 
 Run the Julia unit tests listed in `tests`, which can be either a string or an array of
 strings, using `numcores` processors. If `exit_on_error` is `false`, when one test
 fails, all remaining tests in other files will still be run; they are otherwise discarded,
 when `exit_on_error == true`.
+If a seed is provided via the keyword argument, it is used to seed the
+global RNG in the context where the tests are run; otherwise the seed is chosen randomly.
 """
 function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2);
- exit_on_error=false)
+ exit_on_error=false,
+ seed::Union{BitInteger,Void}=nothing)
  if isa(tests,AbstractString)
  tests = split(tests)
  end
  exit_on_error && push!(tests, "--exit-on-error")
+ seed != nothing && push!(tests, "--seed=0x$(hex(seed % UInt128))") # cast to UInt128 to avoid a minus sign
  ENV2 = copy(ENV)
  ENV2["JULIA_CPU_CORES"] = "$numcores"
  try

test/choosetests.jl

@@ -2,22 +2,29 @@
 
 @doc """
 
-`tests, net_on, exit_on_error = choosetests(choices)` selects a set of tests to be
+`tests, net_on, exit_on_error, seed = choosetests(choices)` selects a set of tests to be
 run. `choices` should be a vector of test names; if empty or set to
 `["all"]`, all tests are selected.
 
 This function also supports "test collections": specifically, "linalg"
  refers to collections of tests in the correspondingly-named
 directories.
 
-Upon return, `tests` is a vector of fully-expanded test names,
-`net_on` is true if networking is available (required for some tests),
-and `exit_on_error` is true if an error in one test should cancel
-remaining tests to be run (otherwise, all tests are run unconditionally).
-
-Two options can be passed to `choosetests` by including a special token
-in the `choices` argument: "--skip", which makes all tests coming after
-be skipped, and "--exit-on-error" which sets the value of `exit_on_error`.
+Upon return:
+ - `tests` is a vector of fully-expanded test names,
+ - `net_on` is true if networking is available (required for some tests),
+ - `exit_on_error` is true if an error in one test should cancel
+ remaining tests to be run (otherwise, all tests are run unconditionally),
+ - `seed` is a seed which will be used to initialize the global RNG for each
+ test to be run.
+
+Three options can be passed to `choosetests` by including a special token
+in the `choices` argument:
+ - "--skip", which makes all tests coming after be skipped,
+ - "--exit-on-error" which sets the value of `exit_on_error`,
+ - "--seed=SEED", which sets the value of `seed` to `SEED`
+ (parsed as an `UInt128`); `seed` is otherwise initialized randomly.
+ This option can be used to reproduce failed tests.
 """ ->
 function choosetests(choices = [])
  testnames = [
@@ -58,13 +65,16 @@ function choosetests(choices = [])
  tests = []
  skip_tests = []
  exit_on_error = false
+ seed = rand(RandomDevice(), UInt128)
 
  for (i, t) in enumerate(choices)
  if t == "--skip"
  skip_tests = choices[i + 1:end]
  break
  elseif t == "--exit-on-error"
  exit_on_error = true
+ elseif startswith(t, "--seed=")
+ seed = parse(UInt128, t[8:end])
  else
  push!(tests, t)
  end
@@ -177,5 +187,5 @@ function choosetests(choices = [])
 
  filter!(x -> !(x in skip_tests), tests)
 
- tests, net_on, exit_on_error
+ tests, net_on, exit_on_error, seed
 end

test/runtests.jl

@@ -4,7 +4,7 @@ using Test
 include("choosetests.jl")
 include("testenv.jl")
 
-tests, net_on, exit_on_error = choosetests(ARGS)
+tests, net_on, exit_on_error, seed = choosetests(ARGS)
 tests = unique(tests)
 
 const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB")
@@ -55,7 +55,7 @@ cd(dirname(@__FILE__)) do
  local resp
  wrkr = p
  try
- resp = remotecall_fetch(runtests, wrkr, test)
+ resp = remotecall_fetch(runtests, wrkr, test; seed=seed)
  catch e
  resp = [e]
  end
@@ -105,7 +105,7 @@ cd(dirname(@__FILE__)) do
  n > 1 && print("\tFrom worker 1:\t")
  local resp
  try
- resp = eval(Expr(:call, () -> runtests(t))) # runtests is defined by the include above
+ resp = eval(Expr(:call, () -> runtests(t, seed=seed))) # runtests is defined by the include above
  catch e
  resp = [e]
  end
@@ -188,7 +188,8 @@ cd(dirname(@__FILE__)) do
  else
  println(" \033[31;1mFAILURE\033[0m\n")
  skipped > 0 &&
- println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.\n")
+ println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.")
+ println("The global RNG seed was 0x$(hex(seed)).\n")
  Test.print_test_errors(o_ts)
  throw(Test.FallbackTestSetException("Test run finished with errors"))
  end

test/testdefs.jl

@@ -2,7 +2,7 @@
 
 using Test
 
-function runtests(name, isolate=true)
+function runtests(name, isolate=true; seed=nothing)
  old_print_setting = Test.TESTSET_PRINT_ENABLE[]
  Test.TESTSET_PRINT_ENABLE[] = false
  try
@@ -17,6 +17,7 @@ function runtests(name, isolate=true)
  @eval(m, using Test)
  ex = quote
  @timed @testset $"$name" begin
+ srand($seed)
  include($"$name.jl")
  end
  end