Support @test_throws just checking the error message (JuliaLang#41888)

With this PR,

    @test_throws "reducing over an empty collection" reduce(+, ())

allows you to check the displayed error message without caring about
the details of which specific Exception subtype is used.

The pattern-matching options are the same as for `@test_warn`.

Co-authored by: Jameson Nash <[email protected]>
Co-authored by: Takafumi Arakaki <[email protected]>

NEWS.md

@@ -37,6 +37,9 @@ New library functions
 New library features
 --------------------
 
+* `@test_throws "some message" triggers_error()` can now be used to check whether the displayed error text
+ contains "some message" regardless of the specific exception type.
+ Regular expressions, lists of strings, and matching functions are also supported. ([#41888)
 
 Standard library changes
 ------------------------

stdlib/Test/src/Test.jl

@@ -86,8 +86,9 @@ struct Pass <: Result
  data
  value
  source::Union{Nothing,LineNumberNode}
- function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing)
- return new(test_type, orig_expr, data, thrown isa String ? "String" : thrown, source)
+ message_only::Bool
+ function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing, message_only=false)
+ return new(test_type, orig_expr, data, thrown, source, message_only)
  end
 end
 
@@ -98,7 +99,11 @@ function Base.show(io::IO, t::Pass)
  end
  if t.test_type === :test_throws
  # The correct type of exception was thrown
- print(io, "\n Thrown: ", t.value isa String ? t.value : typeof(t.value))
+ if t.message_only
+ print(io, "\n Message: ", t.value)
+ else
+ print(io, "\n Thrown: ", typeof(t.value))
+ end
  elseif t.test_type === :test && t.data !== nothing
  # The test was an expression, so display the term-by-term
  # evaluated version as well
@@ -118,12 +123,14 @@ struct Fail <: Result
  data::Union{Nothing, String}
  value::String
  source::LineNumberNode
- function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode)
+ message_only::Bool
+ function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode, message_only::Bool=false)
  return new(test_type,
  string(orig_expr),
  data === nothing ? nothing : string(data),
  string(isa(data, Type) ? typeof(value) : value),
- source)
+ source,
+ message_only)
  end
 end
 
@@ -132,18 +139,24 @@ function Base.show(io::IO, t::Fail)
  print(io, " at ")
  printstyled(io, something(t.source.file, :none), ":", t.source.line, "\n"; bold=true, color=:default)
  print(io, " Expression: ", t.orig_expr)
+ value, data = t.value, t.data
  if t.test_type === :test_throws_wrong
  # An exception was thrown, but it was of the wrong type
- print(io, "\n Expected: ", t.data)
- print(io, "\n Thrown: ", t.value)
+ if t.message_only
+ print(io, "\n Expected: ", data)
+ print(io, "\n Message: ", value)
+ else
+ print(io, "\n Expected: ", data)
+ print(io, "\n Thrown: ", value)
+ end
  elseif t.test_type === :test_throws_nothing
  # An exception was expected, but no exception was thrown
- print(io, "\n Expected: ", t.data)
+ print(io, "\n Expected: ", data)
  print(io, "\n No exception thrown")
- elseif t.test_type === :test && t.data !== nothing
+ elseif t.test_type === :test && data !== nothing
  # The test was an expression, so display the term-by-term
  # evaluated version as well
- print(io, "\n Evaluated: ", t.data)
+ print(io, "\n Evaluated: ", data)
  end
 end
 
@@ -238,6 +251,7 @@ function Serialization.serialize(s::Serialization.AbstractSerializer, t::Pass)
  Serialization.serialize(s, t.data === nothing ? nothing : string(t.data))
  Serialization.serialize(s, string(t.value))
  Serialization.serialize(s, t.source === nothing ? nothing : t.source)
+ Serialization.serialize(s, t.message_only)
  nothing
 end
 
@@ -657,6 +671,8 @@ end
 
 Tests that the expression `expr` throws `exception`.
 The exception may specify either a type,
+a string, regular expression, or list of strings occurring in the displayed error message,
+a matching function,
 or a value (which will be tested for equality by comparing fields).
 Note that `@test_throws` does not support a trailing keyword form.
 
@@ -671,7 +687,18 @@ julia> @test_throws DimensionMismatch [1, 2, 3] + [1, 2]
 Test Passed
  Expression: [1, 2, 3] + [1, 2]
  Thrown: DimensionMismatch
+
+julia> @test_throws "Try sqrt(Complex" sqrt(-1)
+Test Passed
+ Expression: sqrt(-1)
+ Message: "DomainError with -1.0:\\nsqrt will only return a complex result if called with a complex argument. Try sqrt(Complex(x))."
 ```
+
+In the final example, instead of matching a single string it could alternatively have been performed with:
+
+- `["Try", "Complex"]` (a list of strings)
+- `r"Try sqrt\\([Cc]omplex"` (a regular expression)
+- `str -> occursin("complex", str)` (a matching function)
 """
 macro test_throws(extype, ex)
  orig_ex = Expr(:inert, ex)
@@ -697,6 +724,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
  if isa(result, Threw)
  # Check that the right type of exception was thrown
  success = false
+ message_only = false
  exc = result.exception
  # NB: Throwing LoadError from macroexpands is deprecated, but in order to limit
  # the breakage in package tests we add extra logic here.
@@ -712,7 +740,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
  else
  isa(exc, extype)
  end
- else
+ elseif isa(extype, Exception) || !isa(exc, Exception)
  if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc)
  extype = extype.error # deprecated
  end
@@ -725,11 +753,21 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype)
  end
  end
  end
+ else
+ message_only = true
+ exc = sprint(showerror, exc)
+ success = contains_warn(exc, extype)
+ exc = repr(exc)
+ if isa(extype, AbstractString)
+ extype = repr(extype)
+ elseif isa(extype, Function)
+ extype = "< match function >"
+ end
  end
  if success
- testres = Pass(:test_throws, orig_expr, extype, exc, result.source)
+ testres = Pass(:test_throws, orig_expr, extype, exc, result.source, message_only)
  else
- testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source)
+ testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source, message_only)
  end
  else
  testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, result.source)

stdlib/Test/test/runtests.jl

@@ -96,6 +96,16 @@ end
  "Thrown: ErrorException")
  @test endswith(sprint(show, @test_throws ErrorException("test") error("test")),
  "Thrown: ErrorException")
+ @test endswith(sprint(show, @test_throws "a test" error("a test")),
+ "Message: \"a test\"")
+ @test occursin("Message: \"DomainError",
+ sprint(show, @test_throws r"sqrt\([Cc]omplex" sqrt(-1)))
+ @test endswith(sprint(show, @test_throws str->occursin("a t", str) error("a test")),
+ "Message: \"a test\"")
+ @test endswith(sprint(show, @test_throws ["BoundsError", "access", "1-element", "at index [2]"] [1][2]),
+ "Message: \"BoundsError: attempt to access 1-element Vector{$Int} at index [2]\"")
+ @test_throws "\"" throw("\"")
+ @test_throws Returns(false) throw(Returns(false))
 end
 # Test printing of Fail results
 include("nothrow_testset.jl")
@@ -148,6 +158,11 @@ let fails = @testset NoThrowTestSet begin
  @test contains(str1, str2)
  # 22 - Fail - Type Comparison
  @test typeof(1) <: typeof("julia")
+ # 23 - 26 - Fail - wrong message
+ @test_throws "A test" error("a test")
+ @test_throws r"sqrt\([Cc]omplx" sqrt(-1)
+ @test_throws str->occursin("a T", str) error("a test")
+ @test_throws ["BoundsError", "acess", "1-element", "at index [2]"] [1][2]
  end
  for fail in fails
  @test fail isa Test.Fail
@@ -262,6 +277,27 @@ let fails = @testset NoThrowTestSet begin
  @test occursin("Expression: typeof(1) <: typeof(\"julia\")", str)
  @test occursin("Evaluated: $(typeof(1)) <: $(typeof("julia"))", str)
  end
+
+ let str = sprint(show, fails[23])
+ @test occursin("Expected: \"A test\"", str)
+ @test occursin("Message: \"a test\"", str)
+ end
+
+ let str = sprint(show, fails[24])
+ @test occursin("Expected: r\"sqrt\\([Cc]omplx\"", str)
+ @test occursin(r"Message: .*Try sqrt\(Complex", str)
+ end
+
+ let str = sprint(show, fails[25])
+ @test occursin("Expected: < match function >", str)
+ @test occursin("Message: \"a test\"", str)
+ end
+
+ let str = sprint(show, fails[26])
+ @test occursin("Expected: [\"BoundsError\", \"acess\", \"1-element\", \"at index [2]\"]", str)
+ @test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str)
+ end
+
 end
 
 let errors = @testset NoThrowTestSet begin
@@ -1202,4 +1238,4 @@ Test.finish(ts::PassInformationTestSet) = ts
  @test ts.results[2].data == ErrorException
  @test ts.results[2].value == ErrorException("Msg")
  @test ts.results[2].source == LineNumberNode(test_throws_line_number, @__FILE__)
-end
+end