perf: optimize append_any function (#30248)

Reimplement a larger portion of the optimizations in jl_f__apply
in the fallback function, so we can reduce the performance wall in more cases.

General fix for #29133-like performance issues

base/compiler/abstractinterpretation.jl

@@ -149,7 +149,7 @@ function abstract_call_method_with_const_args(@nospecialize(f), argtypes::Vector
  length(argtypes) >= nargs || return Any
  haveconst = false
  for a in argtypes
- a = maybe_widen_conditional(a)
+ a = widenconditional(a)
  if has_nontrivial_const_info(a)
  # have new information from argtypes that wasn't available from the signature
  if !isa(a, Const) || (isa(a.val, Symbol) || isa(a.val, Type) || (!isa(a.val, String) && isimmutable(a.val)))
@@ -181,7 +181,7 @@ function abstract_call_method_with_const_args(@nospecialize(f), argtypes::Vector
  if !istopfunction(f, :getproperty) && !istopfunction(f, :setproperty!)
  # in this case, see if all of the arguments are constants
  for a in argtypes
- a = maybe_widen_conditional(a)
+ a = widenconditional(a)
  if !isa(a, Const) && !isconstType(a)
  return Any
  end
@@ -532,7 +532,7 @@ end
 
 function pure_eval_call(@nospecialize(f), argtypes::Vector{Any}, @nospecialize(atype), sv::InferenceState)
  for i = 2:length(argtypes)
- a = maybe_widen_conditional(argtypes[i])
+ a = widenconditional(argtypes[i])
  if !(isa(a, Const) || isconstType(a))
  return false
  end
@@ -551,7 +551,7 @@ function pure_eval_call(@nospecialize(f), argtypes::Vector{Any}, @nospecialize(a
  return false
  end
 
- args = Any[ (a = maybe_widen_conditional(argtypes[i]); isa(a, Const) ? a.val : a.parameters[1]) for i in 2:length(argtypes) ]
+ args = Any[ (a = widenconditional(argtypes[i]); isa(a, Const) ? a.val : a.parameters[1]) for i in 2:length(argtypes) ]
  try
  value = Core._apply_pure(f, args)
  # TODO: add some sort of edge(s)
@@ -1089,7 +1089,7 @@ function typeinf_local(frame::InferenceState)
  end
  elseif hd === :return
  pc´ = n + 1
- rt = maybe_widen_conditional(abstract_eval(stmt.args[1], s[pc], frame))
+ rt = widenconditional(abstract_eval(stmt.args[1], s[pc], frame))
  if !isa(rt, Const) && !isa(rt, Type) && (!isa(rt, PartialTuple) || frame.cached)
  # only propagate information we know we can store
  # and is valid inter-procedurally

base/compiler/inferenceresult.jl

@@ -36,7 +36,7 @@ function matching_cache_argtypes(linfo::MethodInstance, given_argtypes::Vector)
  @assert isa(linfo.def, Method) # ensure the next line works
  nargs::Int = linfo.def.nargs
  @assert length(given_argtypes) >= (nargs - 1)
- given_argtypes = anymap(maybe_widen_conditional, given_argtypes)
+ given_argtypes = anymap(widenconditional, given_argtypes)
  if linfo.def.isva
  isva_given_argtypes = Vector{Any}(undef, nargs)
  for i = 1:(nargs - 1)

base/compiler/tfuncs.jl

@@ -223,8 +223,8 @@ add_tfunc(ifelse, 3, 3,
  return tmerge(x, y)
  end, 1)
 function egal_tfunc(@nospecialize(x), @nospecialize(y))
- xx = maybe_widen_conditional(x)
- yy = maybe_widen_conditional(y)
+ xx = widenconditional(x)
+ yy = widenconditional(y)
  if isa(x, Conditional) && isa(yy, Const)
  yy.val === false && return Conditional(x.var, x.elsetype, x.vtype)
  yy.val === true && return x
@@ -841,7 +841,7 @@ function apply_type_nothrow(argtypes::Array{Any, 1}, @nospecialize(rt))
  u = headtype
  for i = 2:length(argtypes)
  isa(u, UnionAll) || return false
- ai = maybe_widen_conditional(argtypes[i])
+ ai = widenconditional(argtypes[i])
  if ai === TypeVar
  # We don't know anything about the bounds of this typevar, but as
  # long as the UnionAll is not constrained, that's ok.
@@ -922,7 +922,7 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
  tparams = Any[]
  outervars = Any[]
  for i = 1:largs
- ai = maybe_widen_conditional(args[i])
+ ai = widenconditional(args[i])
  if isType(ai)
  aip1 = ai.parameters[1]
  canconst &= !has_free_typevars(aip1)
@@ -1010,7 +1010,7 @@ end
 # convert the dispatch tuple type argtype to the real (concrete) type of
 # the tuple of those values
 function tuple_tfunc(atypes::Vector{Any})
- atypes = anymap(maybe_widen_conditional, atypes)
+ atypes = anymap(widenconditional, atypes)
  all_are_const = true
  for i in 1:length(atypes)
  if !isa(atypes[i], Const)

base/compiler/typeinfer.jl

@@ -223,17 +223,6 @@ function widen_all_consts!(src::CodeInfo)
  return src
 end
 
-maybe_widen_conditional(@nospecialize vt) = vt
-function maybe_widen_conditional(vt::Conditional)
- if vt.vtype === Bottom
- return Const(false)
- elseif vt.elsetype === Bottom
- return Const(true)
- else
- return Bool
- end
-end
-
 function annotate_slot_load!(e::Expr, vtypes::VarTable, sv::InferenceState, undefs::Array{Bool,1})
  head = e.head
  i0 = 1
@@ -256,7 +245,7 @@ end
 function visit_slot_load!(sl::Slot, vtypes::VarTable, sv::InferenceState, undefs::Array{Bool,1})
  id = slot_id(sl)
  s = vtypes[id]
- vt = maybe_widen_conditional(s.typ)
+ vt = widenconditional(s.typ)
  if s.undef
  # find used-undef variables
  undefs[id] = true
@@ -312,7 +301,7 @@ function type_annotate!(sv::InferenceState)
  if gt[j] === NOT_FOUND
  gt[j] = Union{}
  end
- gt[j] = maybe_widen_conditional(gt[j])
+ gt[j] = widenconditional(gt[j])
  end
 
  # compute the required type for each slot

base/compiler/typelattice.jl

@@ -217,6 +217,7 @@ end
 @inline tchanged(@nospecialize(n), @nospecialize(o)) = o === NOT_FOUND || (n !== NOT_FOUND && !(n ⊑ o))
 @inline schanged(@nospecialize(n), @nospecialize(o)) = (n !== o) && (o === NOT_FOUND || (n !== NOT_FOUND && !issubstate(n, o)))
 
+widenconditional(@nospecialize typ) = typ
 function widenconditional(typ::Conditional)
  if typ.vtype == Union{}
  return Const(false)

base/essentials.jl

@@ -415,27 +415,6 @@ Stacktrace:
 """
 sizeof(x) = Core.sizeof(x)
 
-function append_any(xs...)
- # used by apply() and quote
- # must be a separate function from append(), since apply() needs this
- # exact function.
- out = Vector{Any}(undef, 4)
- l = 4
- i = 1
- for x in xs
- for y in x
- if i > l
- _growend!(out, 16)
- l += 16
- end
- arrayset(true, out, y, i)
- i += 1
- end
- end
- _deleteend!(out, l-i+1)
- out
-end
-
 # simple Array{Any} operations needed for bootstrap
 @eval setindex!(A::Array{Any}, @nospecialize(x), i::Int) = arrayset($(Expr(:boundscheck)), A, x, i)
 
@@ -641,6 +620,72 @@ function isassigned(v::SimpleVector, i::Int)
  return x != C_NULL
 end
 
+
+# used by ... syntax to access the `iterate` function from inside the Core._apply implementation
+# must be a separate function from append(), since Core._apply needs this exact function
+function append_any(xs...)
+ @nospecialize
+ lx = length(xs)
+ l = 4
+ i = 1
+ out = Vector{Any}(undef, l)
+ for xi in 1:lx
+ x = @inbounds xs[xi]
+ # handle some common cases, where we know the length
+ # and can inline the iterator because the runtime
+ # has an optimized version of the iterator
+ if x isa SimpleVector
+ lx = length(x)
+ if i + lx - 1 > l
+ ladd = lx > 16 ? lx : 16
+ _growend!(out, ladd)
+ l += ladd
+ end
+ for j in 1:lx
+ y = @inbounds x[j]
+ arrayset(true, out, y, i)
+ i += 1
+ end
+ elseif x isa Tuple
+ lx = length(x)
+ if i + lx - 1 > l
+ ladd = lx > 16 ? lx : 16
+ _growend!(out, ladd)
+ l += ladd
+ end
+ for j in 1:lx
+ y = @inbounds x[j]
+ arrayset(true, out, y, i)
+ i += 1
+ end
+ elseif x isa Array
+ lx = length(x)
+ if i + lx - 1 > l
+ ladd = lx > 16 ? lx : 16
+ _growend!(out, ladd)
+ l += ladd
+ end
+ for j in 1:lx
+ y = arrayref(true, x, j)
+ arrayset(true, out, y, i)
+ i += 1
+ end
+ else
+ for y in x
+ if i > l
+ _growend!(out, 16)
+ l += 16
+ end
+ arrayset(true, out, y, i)
+ i += 1
+ end
+ end
+ end
+ _deleteend!(out, l - i + 1)
+ return out
+end
+
+
 """
  Colon()
 

src/cgutils.cpp

@@ -1106,7 +1106,7 @@ static std::pair<Value*, bool> emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x,
  }
 
  // intersection with Type needs to be handled specially
- if (jl_has_intersect_type_not_kind(type)) {
+ if (jl_has_intersect_type_not_kind(type) || jl_has_intersect_type_not_kind(intersected_type)) {
  Value *vx = maybe_decay_untracked(boxed(ctx, x));
  Value *vtyp = maybe_decay_untracked(literal_pointer_val(ctx, type));
  if (msg && *msg == "typeassert") {
@@ -1150,6 +1150,16 @@ static std::pair<Value*, bool> emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x,
  return std::make_pair(ctx.builder.CreateICmpEQ(emit_typeof_boxed(ctx, x),
  maybe_decay_untracked(literal_pointer_val(ctx, intersected_type))), false);
  }
+ jl_datatype_t *dt = (jl_datatype_t*)jl_unwrap_unionall(intersected_type);
+ if (jl_is_datatype(dt) && !dt->abstract && jl_subtype(dt->name->wrapper, type)) {
+ // intersection is a supertype of all instances of its constructor,
+ // so the isa test reduces to a comparison of the typename by pointer
+ return std::make_pair(
+ ctx.builder.CreateICmpEQ(
+ mark_callee_rooted(emit_datatype_name(ctx, emit_typeof_boxed(ctx, x))),
+ mark_callee_rooted(literal_pointer_val(ctx, (jl_value_t*)dt->name))),
+ false);
+ }
  // everything else can be handled via subtype tests
  return std::make_pair(ctx.builder.CreateICmpNE(
  ctx.builder.CreateCall(prepare_call(jlsubtype_func),