add a "resilient" option to run all tests unconditionally (JuliaLang#…

…23589)

* add a "resilient" option to run all tests unconditionally

Previously, this was the default: even if one test fails, all
remaining tests are run nonetheless.
Now this requires to pass the "--resilient" option to "test/runtests.jl"
(or `resilient=true` to the `runtest` function); the new default
is to stop running tests as soon as possible when there is one failure.

* rephrase resilient meaning in `runtests` (kshyatt)

* enable --resilient on CI

* change the default, rename resilient -> exit-on-error

* Revert "enable --resilient on CI"

This reverts commit b055f0b.

* add period at the end of sentence

* remove "--eoe" short alias (StefanKarpinski)

base/interactiveutil.jl

@@ -697,15 +697,20 @@ end
 # testing
 
 """
- runtests([tests=["all"] [, numcores=ceil(Int, Sys.CPU_CORES / 2) ]])
+ Base.runtests(tests=["all"], numcores=ceil(Int, Sys.CPU_CORES / 2);
+ exit_on_error=false)
 
 Run the Julia unit tests listed in `tests`, which can be either a string or an array of
-strings, using `numcores` processors. (not exported)
+strings, using `numcores` processors. If `exit_on_error` is `false`, when one test
+fails, all remaining tests in other files will still be run; they are otherwise discarded,
+when `exit_on_error == true`.
 """
-function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2))
+function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2);
+ exit_on_error=false)
  if isa(tests,AbstractString)
  tests = split(tests)
  end
+ exit_on_error && push!(tests, "--exit-on-error")
  ENV2 = copy(ENV)
  ENV2["JULIA_CPU_CORES"] = "$numcores"
  try

test/choosetests.jl

@@ -2,16 +2,22 @@
 
 @doc """
 
-`tests, net_on = choosetests(choices)` selects a set of tests to be
+`tests, net_on, exit_on_error = choosetests(choices)` selects a set of tests to be
 run. `choices` should be a vector of test names; if empty or set to
 `["all"]`, all tests are selected.
 
 This function also supports "test collections": specifically, "linalg"
  refers to collections of tests in the correspondingly-named
 directories.
 
-Upon return, `tests` is a vector of fully-expanded test names, and
-`net_on` is true if networking is available (required for some tests).
+Upon return, `tests` is a vector of fully-expanded test names,
+`net_on` is true if networking is available (required for some tests),
+and `exit_on_error` is true if an error in one test should cancel
+remaining tests to be run (otherwise, all tests are run unconditionally).
+
+Two options can be passed to `choosetests` by including a special token
+in the `choices` argument: "--skip", which makes all tests coming after
+be skipped, and "--exit-on-error" which sets the value of `exit_on_error`.
 """ ->
 function choosetests(choices = [])
  testnames = [
@@ -51,11 +57,14 @@ function choosetests(choices = [])
 
  tests = []
  skip_tests = []
+ exit_on_error = false
 
  for (i, t) in enumerate(choices)
  if t == "--skip"
  skip_tests = choices[i + 1:end]
  break
+ elseif t == "--exit-on-error"
+ exit_on_error = true
  else
  push!(tests, t)
  end
@@ -168,5 +177,5 @@ function choosetests(choices = [])
 
  filter!(x -> !(x in skip_tests), tests)
 
- tests, net_on
+ tests, net_on, exit_on_error
 end

test/runtests.jl

@@ -4,7 +4,7 @@ using Base.Test
 include("choosetests.jl")
 include("testenv.jl")
 
-tests, net_on = choosetests(ARGS)
+tests, net_on, exit_on_error = choosetests(ARGS)
 tests = unique(tests)
 
 const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB")
@@ -33,6 +33,7 @@ cd(dirname(@__FILE__)) do
  n > 1 && addprocs_with_testenv(n)
  BLAS.set_num_threads(1)
  end
+ skipped = 0
 
  @everywhere include("testdefs.jl")
 
@@ -59,14 +60,18 @@ cd(dirname(@__FILE__)) do
  resp = [e]
  end
  push!(results, (test, resp))
- if resp[1] isa Exception || resp[end] > max_worker_rss
+ if resp[1] isa Exception
+ if exit_on_error
+ skipped = length(tests)
+ empty!(tests)
+ end
+ elseif resp[end] > max_worker_rss
  if n > 1
  rmprocs(wrkr, waitfor=30)
  p = addprocs_with_testenv(1)[1]
  remotecall_fetch(include, p, "testdefs.jl")
- else
- # single process testing, bail if mem limit reached
- resp[1] isa Exception || error("Halting tests. Memory limit reached : $resp > $max_worker_rss")
+ else # single process testing
+ error("Halting tests. Memory limit reached : $resp > $max_worker_rss")
  end
  end
  if !isa(resp[1], Exception)
@@ -181,7 +186,9 @@ cd(dirname(@__FILE__)) do
  if !o_ts.anynonpass
  println(" \033[32;1mSUCCESS\033[0m")
  else
- println(" \033[31;1mFAILURE\033[0m")
+ println(" \033[31;1mFAILURE\033[0m\n")
+ skipped > 0 &&
+ println("$skipped test", skipped > 1 ? "s were" : " was", " skipped due to failure.\n")
  Base.Test.print_test_errors(o_ts)
  throw(Test.FallbackTestSetException("Test run finished with errors"))
  end