add NEWS, doc, and optimize rand stuff in at-testset for

NEWS.md

@@ -439,6 +439,10 @@ Library improvements
  definition relies on `ncodeunits` however, so for optimal performance you may need to
  define a custom method for that function.
 
+ * The global RNG is being re-seeded with it own seed at the beginning of each `@testset`,
+ and have its original state restored at the end ([#24445]). This is breaking for testsets
+ relying implicitly on the global RNG being in a specific state.
+
  * `permutedims(m::AbstractMatrix)` is now short for `permutedims(m, (2,1))`, and is now a
  more convenient way of making a "shallow transpose" of a 2D array. This is the
  recommended approach for manipulating arrays of data, rather than the recursively

stdlib/Test/src/Test.jl

@@ -922,6 +922,14 @@ this behavior can be customized in other testset types. If a `for` loop is used
 then the macro collects and returns a list of the return values of the `finish`
 method, which by default will return a list of the testset objects used in
 each iteration.
+
+Before the execution of the body of a `@testset`, there is an implicit
+call to `srand(seed)` where `seed` is the current seed of the global RNG.
+Moreover, after the execution of the body, the state of the global RNG is
+restored to what it was before the `@testset`. This is meant to ease
+reproducibility in case of failure, and to allow seamless
+re-arrangements of `@testset`s regardless of their dependance seedin the
+global RNG.
 """
 macro testset(args...)
  isempty(args) && error("No arguments to @testset")
@@ -1034,26 +1042,28 @@ function testset_forloop(args, testloop, source)
  if !first_iteration
  pop_testset()
  push!(arr, finish(ts))
+ # it's 1000 times faster to copy from tmprng rather than calling srand
+ copy!(Base.GLOBAL_RNG, tmprng)
+
  end
  ts = $(testsettype)($desc; $options...)
  push_testset(ts)
  first_iteration = false
- oldrng = copy(Base.GLOBAL_RNG)
  try
- srand(Base.GLOBAL_RNG.seed)
  $(esc(tests))
  catch err
  # Something in the test block threw an error. Count that as an
  # error in this test set
  record(ts, Error(:nontest_error, :(), err, catch_backtrace(), $(QuoteNode(source))))
- finally
- copy!(Base.GLOBAL_RNG, oldrng)
  end
  end
  quote
  arr = Vector{Any}()
  local first_iteration = true
  local ts
+ local oldrng = copy(Base.GLOBAL_RNG)
+ srand(Base.GLOBAL_RNG.seed)
+ local tmprng = copy(Base.GLOBAL_RNG)
  try
  $(Expr(:for, Expr(:block, [esc(v) for v in loopvars]...), blk))
  finally
@@ -1062,6 +1072,7 @@ function testset_forloop(args, testloop, source)
  pop_testset()
  push!(arr, finish(ts))
  end
+ copy!(Base.GLOBAL_RNG, oldrng)
  end
  arr
  end