Make Vararg not a DataType (JuliaLang#38136)

Currently `Vararg` is a DataType, but is special cased in a
bunch of places to give it special behavior (e.g. in subtyping
and of course in tuple construction). However, unlike all other
DataTypes, it cannot appear as a type parameter, which caused trouble in
PR JuliaLang#38071. Having it be a DataType is a bit of a pun of convenience
in the first place - it's a lot more similar to a tvar (which can
be considered an implementation detail of UnionAll in the same way
Vararg is an implementation detail of Tuple), which has its own
non-type object. This PR does the same to Vararg, and moves it
from being an abstract DataType with special cased behavior to
its own custom type (called `Core.TypeofVararg`). There are a
few small behavior differences, but they are mostly internal.
In particular, we no longer have `Vararg <: Type` and Vararg
objects no longer have the .parameters field. Also, things like
`Vararg{T} where T` are technically illegal now since Vararg is
not a type. However, since a lot of people are using that pattern,
I've brought it back with a deprecation (which is of course off
by default). The only things that's disallowed is `Vararg{N, N} where N`,
but I haven't seen anybody use that.

base/boot.jl

@@ -242,14 +242,23 @@ ccall(:jl_toplevel_eval_in, Any, (Any, Any),
  (f::typeof(Typeof))(x) = ($(_expr(:meta,:nospecialize,:x)); isa(x,Type) ? Type{x} : typeof(x))
  end)
 
-# let the compiler assume that calling Union{} as a constructor does not need
-# to be considered ever (which comes up often as Type{<:T})
-Union{}(a...) = throw(MethodError(Union{}, a))
 
 macro nospecialize(x)
  _expr(:meta, :nospecialize, x)
 end
 
+TypeVar(n::Symbol) = _typevar(n, Union{}, Any)
+TypeVar(n::Symbol, @nospecialize(ub)) = _typevar(n, Union{}, ub)
+TypeVar(n::Symbol, @nospecialize(lb), @nospecialize(ub)) = _typevar(n, lb, ub)
+
+UnionAll(v::TypeVar, @nospecialize(t)) = ccall(:jl_type_unionall, Any, (Any, Any), v, t)
+
+const Vararg = ccall(:jl_toplevel_eval_in, Any, (Any, Any), Core, _expr(:new, TypeofVararg))
+
+# let the compiler assume that calling Union{} as a constructor does not need
+# to be considered ever (which comes up often as Type{<:T})
+Union{}(a...) = throw(MethodError(Union{}, a))
+
 Expr(@nospecialize args...) = _expr(args...)
 
 abstract type Exception end
@@ -378,12 +387,6 @@ mutable struct WeakRef
  (Ptr{Cvoid}, Any), getptls(), v)
 end
 
-TypeVar(n::Symbol) = _typevar(n, Union{}, Any)
-TypeVar(n::Symbol, @nospecialize(ub)) = _typevar(n, Union{}, ub)
-TypeVar(n::Symbol, @nospecialize(lb), @nospecialize(ub)) = _typevar(n, lb, ub)
-
-UnionAll(v::TypeVar, @nospecialize(t)) = ccall(:jl_type_unionall, Any, (Any, Any), v, t)
-
 Tuple{}() = ()
 
 struct VecElement{T}

base/broadcast.jl

@@ -727,8 +727,7 @@ end
  ci = U
  end
  if i == lr && Core.Compiler.isvarargtype(pi)
- N = (Base.unwrap_unionall(pi)::DataType).parameters[2]
- c[i] = Base.rewrap_unionall(Vararg{ci, N}, pi)
+ c[i] = isdefined(pi, :N) ? Vararg{ci, pi.N} : Vararg{ci}
  else
  c[i] = ci
  end

base/combinatorics.jl

@@ -307,7 +307,7 @@ julia> 2^2 * 3^3
 !!! compat "Julia 1.6"
  The method that accepts a tuple requires Julia 1.6 or later.
 """
-function nextprod(a::Union{Tuple{Vararg{<:Integer}},AbstractVector{<:Integer}}, x::Real)
+function nextprod(a::Union{Tuple{Vararg{Integer}},AbstractVector{<:Integer}}, x::Real)
  if x > typemax(Int)
  throw(ArgumentError("unsafe for x > typemax(Int), got $x"))
  end

base/compiler/abstractinterpretation.jl

@@ -1057,10 +1057,11 @@ function sp_type_rewrap(@nospecialize(T), linfo::MethodInstance, isreturn::Bool)
  spsig = linfo.def.sig
  if isa(spsig, UnionAll)
  if !isempty(linfo.sparam_vals)
- env = pointer_from_objref(linfo.sparam_vals) + sizeof(Ptr{Cvoid})
- T = ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), T, spsig, env)
+ sparam_vals = Any[isa(v, Core.TypeofVararg) ? TypeVar(:N, Union{}, Any) :
+ v for v in linfo.sparam_vals]
+ T = ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), T, spsig, sparam_vals)
  isref && isreturn && T === Any && return Bottom # catch invalid return Ref{T} where T = Any
- for v in linfo.sparam_vals
+ for v in sparam_vals
  if isa(v, TypeVar)
  T = UnionAll(v, T)
  end

base/compiler/inferencestate.jl

@@ -192,6 +192,8 @@ function sptypes_from_meth_instance(linfo::MethodInstance)
  ty = UnionAll(tv, Type{tv})
  end
  end
+ elseif isa(v, Core.TypeofVararg)
+ ty = Int
  else
  ty = Const(v)
  end

base/compiler/ssair/inlining.jl

@@ -738,7 +738,7 @@ function analyze_method!(match::MethodMatch, atypes::Vector{Any},
  # Bail out if any static parameters are left as TypeVar
  ok = true
  for i = 1:length(match.sparams)
- isa(match.sparams[i], TypeVar) && return nothing
+ (isa(match.sparams[i], TypeVar) || isa(match.sparams[i], Core.TypeofVararg)) && return nothing
  end
 
  if !params.inlining

base/compiler/tfuncs.jl

@@ -447,9 +447,7 @@ function typebound_nothrow(b)
  b = widenconst(b)
  (b ⊑ TypeVar) && return true
  if isType(b)
- b = unwrap_unionall(b.parameters[1])
- b === Union{} && return true
- return !isa(b, DataType) || b.name != _va_typename
+ return true
  end
  return false
 end
@@ -489,9 +487,8 @@ function typeof_concrete_vararg(t::DataType)
  for i = 1:np
  p = t.parameters[i]
  if i == np && isvarargtype(p)
- pp = unwrap_unionall(p)
- if isconcretetype(pp.parameters[1]) && pp.parameters[2] isa TypeVar
- return rewrap_unionall(Type{Tuple{t.parameters[1:np-1]..., pp}}, p)
+ if isdefined(p, :T) && !isdefined(p, :N) && isconcretetype(p.T)
+ return Type{Tuple{t.parameters[1:np-1]..., Vararg{p.T, N}}} where N
  end
  elseif !isconcretetype(p)
  break
@@ -555,15 +552,21 @@ function typeassert_type_instance(@nospecialize(v), @nospecialize(t))
  widev = widenconst(v)
  if widev <: t
  return v
- elseif typeintersect(widev, t) === Bottom
+ end
+ ti = typeintersect(widev, t)
+ if ti === Bottom
  return Bottom
  end
  @assert widev <: Tuple
  new_fields = Vector{Any}(undef, length(v.fields))
  for i = 1:length(new_fields)
- new_fields[i] = typeassert_type_instance(v.fields[i], getfield_tfunc(t, Const(i)))
- if new_fields[i] === Bottom
- return Bottom
+ if isa(v.fields[i], Core.TypeofVararg)
+ new_fields[i] = v.fields[i]
+ else
+ new_fields[i] = typeassert_type_instance(v.fields[i], getfield_tfunc(t, Const(i)))
+ if new_fields[i] === Bottom
+ return Bottom
+ end
  end
  end
  return tuple_tfunc(new_fields)
@@ -1130,10 +1133,10 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
  elseif isconstType(headtypetype)
  headtype = headtypetype.parameters[1]
  else
- return Type
+ return Any
  end
  if !isempty(args) && isvarargtype(args[end])
- return Type
+ return isvarargtype(headtype) ? Core.TypeofVararg : Type
  end
  largs = length(args)
  if headtype === Union
@@ -1175,8 +1178,8 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
  end
  return allconst ? Const(ty) : Type{ty}
  end
- istuple = (headtype == Tuple)
- if !istuple && !isa(headtype, UnionAll)
+ istuple = isa(headtype, Type) && (headtype == Tuple)
+ if !istuple && !isa(headtype, UnionAll) && !isvarargtype(headtype)
  return Union{}
  end
  uw = unwrap_unionall(headtype)
@@ -1193,7 +1196,8 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
  aip1 = ai.parameters[1]
  canconst &= !has_free_typevars(aip1)
  push!(tparams, aip1)
- elseif isa(ai, Const) && (isa(ai.val, Type) || isa(ai.val, TypeVar) || valid_tparam(ai.val))
+ elseif isa(ai, Const) && (isa(ai.val, Type) || isa(ai.val, TypeVar) ||
+ valid_tparam(ai.val) || (istuple && isa(ai.val, Core.TypeofVararg)))
  push!(tparams, ai.val)
  elseif isa(ai, PartialTypeVar)
  canconst = false
@@ -1259,11 +1263,11 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
  catch ex
  # type instantiation might fail if one of the type parameters
  # doesn't match, which could happen if a type estimate is too coarse
- return Type{<:headtype}
+ return isvarargtype(headtype) ? Core.TypeofVararg : Type{<:headtype}
  end
  !uncertain && canconst && return Const(appl)
- if isvarargtype(headtype)
- return Type
+ if isvarargtype(appl)
+ return Core.TypeofVararg
  end
  if istuple
  return Type{<:appl}
@@ -1315,7 +1319,7 @@ function tuple_tfunc(atypes::Vector{Any})
  x = atypes[i]
  # TODO ignore singleton Const (don't forget to update cache logic if you implement this)
  if !anyinfo
- anyinfo = !isa(x, Type) || isType(x)
+ anyinfo = (!isa(x, Type) && !isvarargtype(x)) || isType(x)
  end
  if isa(x, Const)
  params[i] = typeof(x.val)

base/compiler/typelattice.jl

@@ -220,6 +220,7 @@ widenconst(c::PartialTypeVar) = TypeVar
 widenconst(t::PartialStruct) = t.typ
 widenconst(t::Type) = t
 widenconst(t::TypeVar) = t
+widenconst(t::Core.TypeofVararg) = t
 
 issubstate(a::VarState, b::VarState) = (a.typ ⊑ b.typ && a.undef <= b.undef)
 

base/compiler/typelimits.jl

@@ -46,6 +46,8 @@ function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int)
  # see if it is derived from the body
  # also handle the var here, since this construct bounds the mindepth to the smallest possible value
  return is_derived_type(t, c.var.ub, mindepth) || is_derived_type(t, c.body, mindepth)
+ elseif isa(c, Core.TypeofVararg)
+ return is_derived_type(t, unwrapva(c), mindepth)
  elseif isa(c, DataType)
  if mindepth > 0
  mindepth -= 1
@@ -85,7 +87,7 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
  return t # fast path: unparameterized are always simple
  else
  ut = unwrap_unionall(t)
- if isa(ut, DataType) && ut.name !== _va_typename && isa(c, Type) && c !== Union{} && c <: t
+ if isa(ut, DataType) && isa(c, Type) && c !== Union{} && c <: t
  # TODO: need to check that the UnionAll bounds on t are limited enough too
  return t # t is already wider than the comparison in the type lattice
  elseif is_derived_type_from_any(ut, sources, depth)
@@ -118,19 +120,19 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
  b = _limit_type_size(t.b, c.b, sources, depth, allowed_tuplelen)
  return Union{a, b}
  end
+ elseif isa(t, Core.TypeofVararg)
+ isa(c, Core.TypeofVararg) || return Vararg
+ VaT = _limit_type_size(unwrapva(t), unwrapva(c), sources, depth + 1, 0)
+ if isdefined(t, :N) && (isa(t.N, TypeVar) || (isdefined(c, :N) && t.N === c.N))
+ return Vararg{VaT, t.N}
+ end
+ return Vararg{VaT}
  elseif isa(t, DataType)
  if isa(c, DataType)
  tP = t.parameters
  cP = c.parameters
  if t.name === c.name && !isempty(cP)
- if isvarargtype(t)
- VaT = _limit_type_size(tP[1], cP[1], sources, depth + 1, 0)
- N = tP[2]
- if isa(N, TypeVar) || N === cP[2]
- return Vararg{VaT, N}
- end
- return Vararg{VaT}
- elseif t.name === Tuple.name
+ if t.name === Tuple.name
  # for covariant datatypes (Tuple),
  # apply type-size limit element-wise
  ltP = length(tP)
@@ -155,21 +157,17 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
  end
  return Tuple{Q...}
  end
- elseif isvarargtype(c)
- # Tuple{Vararg{T}} --> Tuple{T} is OK
- return _limit_type_size(t, cP[1], sources, depth, 0)
  end
+ elseif isa(c, Core.TypeofVararg)
+ # Tuple{Vararg{T}} --> Tuple{T} is OK
+ return _limit_type_size(t, c.T, sources, depth, 0)
  end
  if isType(t) # allow taking typeof as Type{...}, but ensure it doesn't start nesting
  tt = unwrap_unionall(t.parameters[1])
  if isa(tt, DataType) && !isType(tt)
  is_derived_type_from_any(tt, sources, depth) && return t
  end
  end
- if isvarargtype(t)
- # never replace Vararg with non-Vararg
- return Vararg
- end
  if allowed_tuplelen < 1 && t.name === Tuple.name
  return Any
  end

base/compiler/typeutils.jl

@@ -61,6 +61,17 @@ function valid_tparam(@nospecialize(x))
  return isa(x, Symbol) || isbits(x)
 end
 
+function compatible_vatuple(a::DataType, b::DataType)
+ vaa = a.parameters[end]
+ vab = a.parameters[end]
+ if !(isa(vaa, Core.TypeofVararg) && isa(vab, Core.TypeofVararg))
+ return isa(vaa, Core.TypeofVararg) == isa(vab, Core.TypeofVararg)
+ end
+ (isdefined(vaa, :N) == isdefined(vab, :N)) || return false
+ !isdefined(vaa, :N) && return true
+ return vaa.N === vab.N
+end
+
 # return an upper-bound on type `a` with type `b` removed
 # such that `return <: a` && `Union{return, b} == Union{a, b}`
 function typesubtract(@nospecialize(a), @nospecialize(b), MAX_UNION_SPLITTING::Int)
@@ -80,19 +91,23 @@ function typesubtract(@nospecialize(a), @nospecialize(b), MAX_UNION_SPLITTING::I
  ta = switchtupleunion(a)
  return typesubtract(Union{ta...}, b, 0)
  elseif b isa DataType
+ if !compatible_vatuple(a, b)
+ return a
+ end
  # if exactly one element is not bottom after calling typesubtract
  # then the result is all of the elements as normal except that one
  notbottom = fill(false, length(a.parameters))
  for i = 1:length(notbottom)
- ap = a.parameters[i]
- bp = b.parameters[i]
+ ap = unwrapva(a.parameters[i])
+ bp = unwrapva(b.parameters[i])
  notbottom[i] = !(ap <: bp && isnotbrokensubtype(ap, bp))
  end
  let i = findfirst(notbottom)
  if i !== nothing && findnext(notbottom, i + 1) === nothing
  ta = collect(a.parameters)
  ap = a.parameters[i]
  bp = b.parameters[i]
+ (isa(ap, Core.TypeofVararg) || isa(bp, Core.TypeofVararg)) && return a
  ta[i] = typesubtract(ap, bp, min(2, MAX_UNION_SPLITTING))
  return Tuple{ta...}
  end
@@ -196,6 +211,7 @@ function unioncomplexity(t::DataType)
  return c
 end
 unioncomplexity(u::UnionAll) = max(unioncomplexity(u.body), unioncomplexity(u.var.ub))
+unioncomplexity(t::Core.TypeofVararg) = isdefined(t, :T) ? unioncomplexity(t.T) : 0
 unioncomplexity(@nospecialize(x)) = 0
 
 function improvable_via_constant_propagation(@nospecialize(t))

base/docs/basedocs.jl

@@ -2346,7 +2346,7 @@ arguments accepted by varargs methods (see the section on [Varargs Functions](@r
 # Examples
 ```jldoctest
 julia> mytupletype = Tuple{AbstractString, Vararg{Int}}
-Tuple{AbstractString, Vararg{Int64, N} where N}
+Tuple{AbstractString, Vararg{Int64}}
 
 julia> isa(("1",), mytupletype)
 true

base/errorshow.jl

@@ -423,7 +423,7 @@ function show_method_candidates(io::IO, ex::MethodError, @nospecialize kwargs=()
  # If isvarargtype then it checks whether the rest of the input arguments matches
  # the varargtype
  if Base.isvarargtype(sig[i])
- sigstr = (unwrap_unionall(sig[i]).parameters[1], "...")
+ sigstr = (unwrap_unionall(sig[i]).T, "...")
  j = length(t_i)
  else
  sigstr = (sig[i],)
@@ -460,7 +460,7 @@ function show_method_candidates(io::IO, ex::MethodError, @nospecialize kwargs=()
  # It ensures that methods like f(a::AbstractString...) gets the correct
  # number of right_matches
  for t in arg_types_param[length(sig):end]
- if t <: rewrap_unionall(unwrap_unionall(sig[end]).parameters[1], method.sig)
+ if t <: rewrap_unionall(unwrap_unionall(sig[end]).T, method.sig)
  right_matches += 1
  end
  end
@@ -473,7 +473,7 @@ function show_method_candidates(io::IO, ex::MethodError, @nospecialize kwargs=()
  for (k, sigtype) in enumerate(sig[length(t_i)+1:end])
  sigtype = isvarargtype(sigtype) ? unwrap_unionall(sigtype) : sigtype
  if Base.isvarargtype(sigtype)
- sigstr = ((sigtype::DataType).parameters[1], "...")
+ sigstr = ((sigtype::Core.TypeofVararg).T, "...")
  else
  sigstr = (sigtype,)
  end