Merge pull request JuliaLang#22315 from JuliaLang/cv/test-fixes

Fix issues with showing failed `@test` results

base/test.jl

@@ -188,19 +188,22 @@ struct Threw <: ExecutionResult
  backtrace
 end
 
-function eval_comparison(ex::Expr)
+function eval_comparison(evaluated::Expr, quoted::Expr)
  res = true
  i = 1
- a = ex.args
- n = length(a)
+ args = evaluated.args
+ quoted_args = quoted.args
+ n = length(args)
  while i < n
- res = a[i+1](a[i], a[i+2])
- if !isa(res,Bool) || !res
- break
+ a, op, b = args[i], args[i+1], args[i+2]
+ if res
+ res = op(a, b) === true # Keep `res` type stable
  end
+ quoted_args[i] = a
+ quoted_args[i+2] = b
  i += 2
  end
- Returned(res, ex)
+ Returned(res, quoted)
 end
 
 const comparison_prec = Base.operator_precedence(:(==))
@@ -295,18 +298,19 @@ end
 function get_test_result(ex)
  orig_ex = Expr(:inert, ex)
  # Normalize non-dot comparison operator calls to :comparison expressions
- if isa(ex, Expr) && ex.head == :call && length(ex.args)==3 &&
+ if isa(ex, Expr) && ex.head == :call && length(ex.args) == 3 &&
  first(string(ex.args[1])) != '.' &&
  (ex.args[1] === :(==) || Base.operator_precedence(ex.args[1]) == comparison_prec)
- testret = :(eval_comparison(Expr(:comparison,
-  $(esc(ex.args[2])), $(esc(ex.args[1])), $(esc(ex.args[3])))))
- elseif isa(ex, Expr) && ex.head == :comparison
+ ex = Expr(:comparison, ex.args[2], ex.args[1], ex.args[3])
+ end
+ if isa(ex, Expr) && ex.head == :comparison
  # pass all terms of the comparison to `eval_comparison`, as an Expr
- terms = ex.args
- for i = 1:length(terms)
- terms[i] = esc(terms[i])
- end
- testret = :(eval_comparison(Expr(:comparison, $(terms...))))
+ escaped_terms = [esc(arg) for arg in ex.args]
+ quoted_terms = [QuoteNode(arg) for arg in ex.args]
+ testret = :(eval_comparison(
+ Expr(:comparison, $(escaped_terms...)),
+ Expr(:comparison, $(quoted_terms...)),
+ ))
  else
  testret = :(Returned($(esc(ex)), nothing))
  end

test/test.jl

@@ -7,6 +7,12 @@
 @test strip("\t hi \n") == "hi"
 @test strip("\t this should fail \n") != "hi"
 
+# @test should only evaluate the arguments once
+let g = Int[], f = (x) -> (push!(g, x); x)
+ @test f(1) == 1
+ @test g == [1]
+end
+
 # Test @test_broken with fail
 @test_broken false
 @test_broken 1 == 2
@@ -60,16 +66,46 @@ fails = @testset NoThrowTestSet begin
  @test_throws OverflowError 1 + 1
  # Fail - comparison
  @test 1+1 == 2+2
+ # Fail - approximate comparison
+ @test 1/1 ≈ 2/1
+ # Fail - chained comparison
+ @test 1+0 == 2+0 == 3+0
+ # Fail - comparison call
+ @test ==(1 - 2, 2 - 1)
  # Error - unexpected pass
  @test_broken true
 end
-for i in 1:3
+for i in 1:length(fails) - 1
  @test isa(fails[i], Base.Test.Fail)
 end
-@test contains(sprint(show, fails[1]), "Thrown: ErrorException")
-@test contains(sprint(show, fails[2]), "No exception thrown")
-@test contains(sprint(show, fails[3]), "Evaluated: 2 == 4")
-@test contains(sprint(show, fails[4]), "Unexpected Pass")
+
+str = sprint(show, fails[1])
+@test contains(str, "Expression: error()")
+@test contains(str, "Thrown: ErrorException")
+
+str = sprint(show, fails[2])
+@test contains(str, "Expression: 1 + 1")
+@test contains(str, "No exception thrown")
+
+str = sprint(show, fails[3])
+@test contains(str, "Expression: 1 + 1 == 2 + 2")
+@test contains(str, "Evaluated: 2 == 4")
+
+str = sprint(show, fails[4])
+@test contains(str, "Expression: 1 / 1 ≈ 2 / 1")
+@test contains(str, "Evaluated: 1.0 ≈ 2.0")
+
+str = sprint(show, fails[5])
+@test contains(str, "Expression: 1 + 0 == 2 + 0 == 3 + 0")
+@test contains(str, "Evaluated: 1 == 2 == 3")
+
+str = sprint(show, fails[6])
+@test contains(str, "Expression: 1 - 2 == 2 - 1")
+@test contains(str, "Evaluated: -1 == 1")
+
+str = sprint(show, fails[7])
+@test contains(str, "Unexpected Pass")
+@test contains(str, "Expression: true")
 
 # Test printing of a TestSetException
 tse_str = sprint(show, Test.TestSetException(1,2,3,4,Vector{Union{Base.Test.Error, Base.Test.Fail}}()))