Merge pull request #14406 from JuliaLang/jb/fix14009

fix #14009; check for types growing due to recursion and widen them

base/inference.jl

@@ -208,6 +208,18 @@ add_tfunc(typeof, 1, 1, typeof_tfunc)
 add_tfunc(typeassert, 2, 2,
  (A, v, t)->(isType(t) ? typeintersect(v,t.parameters[1]) : Any))
 
+function type_depth(t::ANY, d::Int=0)
+ if isa(t,Union)
+ t === Bottom && return d
+ return maximum(x->type_depth(x, d+1), t.types)
+ elseif isa(t,DataType)
+ P = t.parameters
+ isempty(P) && return d
+ return maximum(x->type_depth(x, d+1), P)
+ end
+ return d
+end
+
 function limit_type_depth(t::ANY, d::Int, cov::Bool, vars)
  if isa(t,TypeVar) || isa(t,TypeConstructor)
  return t
@@ -1508,6 +1520,20 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
  #print("typeinf ", ast0, " ", sparams, " ", atypes, "\n")
 
  global inference_stack, CYCLE_ID
+
+ f = inference_stack
+ while !isa(f,EmptyCallStack)
+ if is(f.ast,ast0)
+ # impose limit if we recur and the argument types grow beyond MAX_TYPE_DEPTH
+ td = type_depth(atypes)
+ if td > MAX_TYPE_DEPTH && td > type_depth(f.types)
+ atypes = limit_type_depth(atypes, 0, true, [])
+ break
+ end
+ end
+ f = f.prev
+ end
+
  # check for recursion
  f = inference_stack
  while !isa(f,EmptyCallStack)

test/core.jl

@@ -3586,3 +3586,13 @@ function __f_isa_arg_1()
  length(a)
 end
 @test __f_isa_arg_1() == 1
+
+# non-terminating inference, issue #14009
+type A14009{T}; end
+A14009{T}(a::T) = A14009{T}()
+f14009(a) = rand(Bool) ? f14009(A14009(a)) : a
+code_typed(f14009, (Int,))
+
+type B14009{T}; end
+g14009(a) = g14009(B14009{a})
+code_typed(g14009, (Type{Int},))