[Test] Define broken/skip keyword argument for @test

NEWS.md

@@ -57,6 +57,21 @@ Standard library changes
 * `keys(::RegexMatch)` is now defined to return the capture's keys, by name if named, or by index if not ([#37299]).
 * `keys(::Generator)` is now defined to return the iterator's keys ([#34678])
 * `RegexMatch` now iterate to give their captures. ([#34355]).
+* `Test.@test` now accepts `broken` and `skip` boolean keyword arguments, which
+ mimic `Test.@test_broken` and `Test.@test_skip` behavior, but allows skipping
+ tests failing only under certain conditions. For example
+ ```julia
+ if T == Float64
+ @test_broken isequal(complex(one(T)) / complex(T(Inf), T(-Inf)), complex(zero(T), zero(T)))
+ else
+ @test isequal(complex(one(T)) / complex(T(Inf), T(-Inf)), complex(zero(T), zero(T)))
+ end
+ ```
+ can be replaced by
+ ```julia
+ @test isequal(complex(one(T)) / complex(T(Inf), T(-Inf)), complex(zero(T), zero(T))) broken=(T == Float64)
+ ```
+ ([#39322])
 
 #### Package Manager
 

stdlib/Test/src/Test.jl

@@ -342,6 +342,8 @@ end
 """
  @test ex
  @test f(args...) key=val ...
+ @test ex broken=true
+ @test ex skip=true
 
 Tests that the expression `ex` evaluates to `true`.
 Returns a `Pass` `Result` if it does, a `Fail` `Result` if it is
@@ -368,12 +370,71 @@ Test Passed
 This is equivalent to the uglier test `@test ≈(π, 3.14, atol=0.01)`.
 It is an error to supply more than one expression unless the first
 is a call expression and the rest are assignments (`k=v`).
+
+You can use any key for the `key=val` arguments, except for `broken` and `skip`,
+which have special meanings in the context of `@test`:
+
+* `broken=cond` indicates a test that should pass but currently consistently
+ fails when `cond==true`. Tests that the expression `ex` evaluates to `false`
+ or causes an exception. Returns a `Broken` `Result` if it does, or an `Error`
+ `Result` if the expression evaluates to `true`. Regular `@test ex` is
+ evaluated when `cond==false`.
+* `skip=cond` marks a test that should not be executed but should be included in
+ test summary reporting as `Broken`, when `cond==true`. This can be useful for
+ tests that intermittently fail, or tests of not-yet-implemented functionality.
+ Regular `@test ex` is evaluated when `cond==false`.
+
+# Examples
+
+```jldoctest
+julia> @test 2 + 2 ≈ 6 atol=1 broken=true
+Test Broken
+ Expression: ≈(2 + 2, 6, atol = 1)
+
+julia> @test 2 + 2 ≈ 5 atol=1 broken=false
+Test Passed
+
+julia> @test 2 + 2 == 5 skip=true
+Test Broken
+ Skipped: 2 + 2 == 5
+
+julia> @test 2 + 2 == 4 skip=false
+Test Passed
+```
+
+!!! compat "Julia 1.7"
+ The `broken` and `skip` keyword arguments require at least Julia 1.7.
 """
 macro test(ex, kws...)
+ # Collect the broken/skip keywords and remove them from the rest of keywords
+ broken = [kw.args[2] for kw in kws if kw.args[1] === :broken]
+ skip = [kw.args[2] for kw in kws if kw.args[1] === :skip]
+ kws = filter(kw -> kw.args[1] ∉ (:skip, :broken), kws)
+ # Validation of broken/skip keywords
+ for (kw, name) in ((broken, :broken), (skip, :skip))
+ if length(kw) > 1
+ error("invalid test macro call: cannot set $(name) keyword multiple times")
+ end
+ end
+ if length(skip) > 0 && length(broken) > 0
+ error("invalid test macro call: cannot set both skip and broken keywords")
+ end
+
+ # Build the test expression
  test_expr!("@test", ex, kws...)
  orig_ex = Expr(:inert, ex)
+
  result = get_test_result(ex, __source__)
- :(do_test($result, $orig_ex))
+
+ return quote
+ if $(length(skip) > 0 && esc(skip[1]))
+ record(get_testset(), Broken(:skipped, $orig_ex))
+ else
+ let _do = $(length(broken) > 0 && esc(broken[1])) ? do_broken_test : do_test
+ _do($result, $orig_ex)
+ end
+ end
+ end
 end
 
 """
@@ -383,7 +444,8 @@ end
 Indicates a test that should pass but currently consistently fails.
 Tests that the expression `ex` evaluates to `false` or causes an
 exception. Returns a `Broken` `Result` if it does, or an `Error` `Result`
-if the expression evaluates to `true`.
+if the expression evaluates to `true`. This is equivalent to
+[`@test ex broken=true`](@ref @test).
 
 The `@test_broken f(args...) key=val...` form works as for the `@test` macro.
 
@@ -412,7 +474,8 @@ end
 
 Marks a test that should not be executed but should be included in test
 summary reporting as `Broken`. This can be useful for tests that intermittently
-fail, or tests of not-yet-implemented functionality.
+fail, or tests of not-yet-implemented functionality. This is equivalent to
+[`@test ex skip=true`](@ref @test).
 
 The `@test_skip f(args...) key=val...` form works as for the `@test` macro.
 

stdlib/Test/test/runtests.jl

@@ -25,6 +25,18 @@ import Logging: Debug, Info, Warn
  @test isapprox(1, 2; atol)
  @test isapprox(1, 3; a.atol)
 end
+@testset "@test with skip/broken kwargs" begin
+ # Make sure the local variables can be used in conditions
+ a = 1
+ @test 2 + 2 == 4 broken=false
+ @test error() broken=true
+ @test !Sys.iswindows() broken=Sys.iswindows()
+ @test 1 ≈ 2 atol=1 broken=a==2
+ @test false skip=true
+ @test true skip=false
+ @test Grogu skip=isone(a)
+ @test 41 ≈ 42 rtol=1 skip=false
+end
 @testset "@test keyword precedence" begin
  atol = 2
  # post-semicolon keyword, suffix keyword, pre-semicolon keyword

test/complex.jl

@@ -1088,16 +1088,8 @@ end
  @test isequal(one(T) / complex(T(-NaN), T(-Inf)), complex(-zero(T), zero(T)))
 
  # divide complex by complex Inf
- if T == Float64
- @test_broken isequal(complex(one(T)) / complex(T(Inf), T(-Inf)), complex(zero(T), zero(T)))
- @test_broken isequal(complex(one(T)) / complex(T(-Inf), T(Inf)), complex(-zero(T), -zero(T)))
- elseif T == Float32
- @test isequal(complex(one(T)) / complex(T(Inf), T(-Inf)), complex(zero(T), zero(T)))
- @test_broken isequal(complex(one(T)) / complex(T(-Inf), T(Inf)), complex(-zero(T), -zero(T)))
- else
- @test isequal(complex(one(T)) / complex(T(Inf), T(-Inf)), complex(zero(T), zero(T)))
- @test isequal(complex(one(T)) / complex(T(-Inf), T(Inf)), complex(-zero(T), -zero(T)))
- end
+ @test isequal(complex(one(T)) / complex(T(Inf), T(-Inf)), complex(zero(T), zero(T))) broken=(T==Float64)
+ @test isequal(complex(one(T)) / complex(T(-Inf), T(Inf)), complex(-zero(T), -zero(T))) broken=(T in (Float32, Float64))
  end
 end
 

test/ranges.jl

@@ -618,14 +618,10 @@ end
  @test broadcast(+, T(1):2:6, 0.3) === T(1)+0.3:2:5+0.3
  @test broadcast(-, T(1):2:6, 1) === T(0):2:4
  @test broadcast(-, T(1):2:6, 0.3) === T(1)-0.3:2:5-0.3
- if T <: Unsigned
- @test_broken broadcast(-, T(1):3) == -T(1):-1:-T(3)
- @test_broken broadcast(-, 2, T(1):3) == T(1):-1:-T(1)
- else
- @test length(broadcast(-, T(1):3, 2)) === length(T(1)-2:T(3)-2)
- @test broadcast(-, T(1):3) == -T(1):-1:-T(3)
- @test broadcast(-, 2, T(1):3) == T(1):-1:-T(1)
- end
+ is_unsigned = T <: Unsigned
+ is_unsigned && @test length(broadcast(-, T(1):3, 2)) === length(T(1)-2:T(3)-2)
+ @test broadcast(-, T(1):3) == -T(1):-1:-T(3) broken=is_unsigned
+ @test broadcast(-, 2, T(1):3) == T(1):-1:-T(1) broken=is_unsigned
 end
 @testset "operations between ranges and arrays" for T in (Int, UInt, Int128)
  @test all(([T(1):5;] + (T(5):-1:1)) .=== T(6))
@@ -1569,17 +1565,11 @@ end
 
 @testset "constant-valued ranges (issues #10391 and #29052)" begin
  for r in ((1:4), (1:1:4), (1.0:4.0))
- if eltype(r) === Int
- @test_broken @inferred(0 * r) == [0.0, 0.0, 0.0, 0.0]
- @test_broken @inferred(0 .* r) == [0.0, 0.0, 0.0, 0.0]
- @test_broken @inferred(r + (4:-1:1)) == [5.0, 5.0, 5.0, 5.0]
- @test_broken @inferred(r .+ (4:-1:1)) == [5.0, 5.0, 5.0, 5.0]
- else
- @test @inferred(0 * r) == [0.0, 0.0, 0.0, 0.0]
- @test @inferred(0 .* r) == [0.0, 0.0, 0.0, 0.0]
- @test @inferred(r + (4:-1:1)) == [5.0, 5.0, 5.0, 5.0]
- @test @inferred(r .+ (4:-1:1)) == [5.0, 5.0, 5.0, 5.0]
- end
+ is_int = eltype(r) === Int
+ @test @inferred(0 * r) == [0.0, 0.0, 0.0, 0.0] broken=is_int
+ @test @inferred(0 .* r) == [0.0, 0.0, 0.0, 0.0] broken=is_int
+ @test @inferred(r + (4:-1:1)) == [5.0, 5.0, 5.0, 5.0] broken=is_int
+ @test @inferred(r .+ (4:-1:1)) == [5.0, 5.0, 5.0, 5.0] broken=is_int
  @test @inferred(r .+ (4.0:-1:1)) == [5.0, 5.0, 5.0, 5.0]
  @test @inferred(0.0 * r) == [0.0, 0.0, 0.0, 0.0]
  @test @inferred(0.0 .* r) == [0.0, 0.0, 0.0, 0.0]