add a "resilient" option to run all tests unconditionally

Previously, this was the default: even if one test fails, all
remaining tests are run nonetheless.
Now this requires to pass the "--resilient" option to "test/runtests.jl"
(or `resilient=true` to the `runtest` function); the new default
is to stop running tests as soon as possible when there is one failure.

base/interactiveutil.jl

@@ -700,15 +700,19 @@ end
 # testing
 
 """
- runtests([tests=["all"] [, numcores=ceil(Int, Sys.CPU_CORES / 2) ]])
+ Base.runtests(tests=["all"], numcores=ceil(Int, Sys.CPU_CORES / 2);
+ resilient=false)
 
 Run the Julia unit tests listed in `tests`, which can be either a string or an array of
-strings, using `numcores` processors. (not exported)
+strings, using `numcores` processors.
+If one test fails, all remaining tests will be discarded, unless resilient is `true`.
 """
-function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2))
+function runtests(tests = ["all"], numcores = ceil(Int, Sys.CPU_CORES / 2);
+ resilient=false)
  if isa(tests,AbstractString)
  tests = split(tests)
  end
+ resilient && push!(tests, "--resilient")
  ENV2 = copy(ENV)
  ENV2["JULIA_CPU_CORES"] = "$numcores"
  try

test/choosetests.jl

@@ -2,16 +2,18 @@
 
 @doc """
 
-`tests, net_on = choosetests(choices)` selects a set of tests to be
+`tests, net_on, resilient = choosetests(choices)` selects a set of tests to be
 run. `choices` should be a vector of test names; if empty or set to
 `["all"]`, all tests are selected.
 
 This function also supports "test collections": specifically, "linalg"
  refers to collections of tests in the correspondingly-named
 directories.
 
-Upon return, `tests` is a vector of fully-expanded test names, and
-`net_on` is true if networking is available (required for some tests).
+Upon return, `tests` is a vector of fully-expanded test names,
+`net_on` is true if networking is available (required for some tests),
+and `resilient` is true if all tests should be run unconditionally
+(.i.e. even when some of them fail).
 """ ->
 function choosetests(choices = [])
  testnames = [
@@ -51,11 +53,14 @@ function choosetests(choices = [])
 
  tests = []
  skip_tests = []
+ resilient = false
 
  for (i, t) in enumerate(choices)
  if t == "--skip"
  skip_tests = choices[i + 1:end]
  break
+ elseif t == "--resilient"
+ resilient = true
  else
  push!(tests, t)
  end
@@ -168,5 +173,5 @@ function choosetests(choices = [])
 
  filter!(x -> !(x in skip_tests), tests)
 
- tests, net_on
+ tests, net_on, resilient
 end

test/runtests.jl

@@ -4,7 +4,7 @@ using Base.Test
 include("choosetests.jl")
 include("testenv.jl")
 
-tests, net_on = choosetests(ARGS)
+tests, net_on, resilient = choosetests(ARGS)
 tests = unique(tests)
 
 const max_worker_rss = if haskey(ENV, "JULIA_TEST_MAXRSS_MB")
@@ -33,6 +33,7 @@ cd(dirname(@__FILE__)) do
  n > 1 && addprocs_with_testenv(n)
  BLAS.set_num_threads(1)
  end
+ skipped = 0
 
  @everywhere include("testdefs.jl")
 
@@ -59,14 +60,18 @@ cd(dirname(@__FILE__)) do
  resp = [e]
  end
  push!(results, (test, resp))
- if resp[1] isa Exception || resp[end] > max_worker_rss
+ if resp[1] isa Exception
+ if !resilient
+ skipped = length(tests)
+ empty!(tests)
+ end
+ elseif resp[end] > max_worker_rss
  if n > 1
  rmprocs(wrkr, waitfor=30)
  p = addprocs_with_testenv(1)[1]
  remotecall_fetch(include, p, "testdefs.jl")
- else
- # single process testing, bail if mem limit reached
- resp[1] isa Exception || error("Halting tests. Memory limit reached : $resp > $max_worker_rss")
+ else # single process testing
+ error("Halting tests. Memory limit reached : $resp > $max_worker_rss")
  end
  end
  if !isa(resp[1], Exception)
@@ -181,7 +186,10 @@ cd(dirname(@__FILE__)) do
  if !o_ts.anynonpass
  println(" \033[32;1mSUCCESS\033[0m")
  else
- println(" \033[31;1mFAILURE\033[0m")
+ println(" \033[31;1mFAILURE\033[0m\n")
+ skipped > 0 &&
+ println("$skipped test", skipped > 1 ? "s were" : " was",
+ """ skipped due to failure (use the "resilient" option to run all tests unconditionally)\n""")
  Base.Test.print_test_errors(o_ts)
  throw(Test.FallbackTestSetException("Test run finished with errors"))
  end