fix #40048, stack overflows in type intersection

src/subtype.c

@@ -2210,13 +2210,29 @@ static int subtype_in_env_existential(jl_value_t *x, jl_value_t *y, jl_stenv_t *
  return issub;
 }
 
+// See if var y is reachable from x via bounds; used to avoid cycles.
+static int reachable_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e)
+{
+ if (in_union(x, (jl_value_t*)y))
+ return 1;
+ if (!jl_is_typevar(x))
+ return 0;
+ jl_varbinding_t *xv = lookup(e, (jl_tvar_t*)x);
+ if (xv == NULL)
+ return 0;
+ return reachable_var(xv->ub, y, e) || reachable_var(xv->lb, y, e);
+}
+
 static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int8_t R, int param)
 {
  jl_varbinding_t *bb = lookup(e, b);
  if (bb == NULL)
  return R ? intersect_aside(a, b->ub, e, 1, 0) : intersect_aside(b->ub, a, e, 0, 0);
- if (bb->lb == bb->ub && jl_is_typevar(bb->lb) && bb->lb != (jl_value_t*)b)
+ if (reachable_var(bb->lb, b, e) || reachable_var(bb->ub, b, e))
+ return a;
+ if (bb->lb == bb->ub && jl_is_typevar(bb->lb)) {
  return intersect(a, bb->lb, e, param);
+ }
  if (!jl_is_type(a) && !jl_is_typevar(a))
  return set_var_to_const(bb, a, NULL);
  int d = bb->depth0;
@@ -2521,7 +2537,11 @@ static jl_value_t *intersect_unionall_(jl_value_t *t, jl_unionall_t *u, jl_stenv
  // if the var for this unionall (based on identity) already appears somewhere
  // in the environment, rename to get a fresh var.
  // TODO: might need to look inside types in btemp->lb and btemp->ub
+ int envsize = 0;
  while (btemp != NULL) {
+ envsize++;
+ if (envsize > 150)
+ return t;
  if (btemp->var == u->var || btemp->lb == (jl_value_t*)u->var ||
  btemp->ub == (jl_value_t*)u->var) {
  u = rename_unionall(u);
@@ -2923,19 +2943,6 @@ static int subtype_by_bounds(jl_value_t *x, jl_value_t *y, jl_stenv_t *e)
  return compareto_var(x, (jl_tvar_t*)y, e, -1) || compareto_var(y, (jl_tvar_t*)x, e, 1);
 }
 
-// See if var y is reachable from x via bounds; used to avoid cycles.
-static int reachable_var(jl_value_t *x, jl_tvar_t *y, jl_stenv_t *e)
-{
- if (x == (jl_value_t*)y)
- return 1;
- if (!jl_is_typevar(x))
- return 0;
- jl_varbinding_t *xv = lookup(e, (jl_tvar_t*)x);
- if (xv == NULL)
- return 0;
- return reachable_var(xv->ub, y, e) || reachable_var(xv->lb, y, e);
-}
-
 // `param` means we are currently looking at a parameter of a type constructor
 // (as opposed to being outside any type constructor, or comparing variable bounds).
 // this is used to record the positions where type variables occur for the

test/subtype.jl

@@ -1914,3 +1914,16 @@ end
 f18985(x::T, y...) where {T<:Union{Int32,Int64}} = (length(y), f18985(y[1], y[2:end]...)...)
 f18985(x::T) where {T<:Union{Int32,Int64}} = 100
 @test f18985(1, 2, 3) == (2, 1, 100)
+
+# issue #40048
+let A = Tuple{Ref{T}, Vararg{T}} where T,
+ B = Tuple{Ref{U}, Union{Ref{S}, Ref{U}, Int}, Union{Ref{S}, S}} where S where U,
+ C = Tuple{Ref{U}, Union{Ref{S}, Ref{U}, Ref{W}}, Union{Ref{S}, W, V}} where V<:AbstractArray where W where S where U
+ I = typeintersect(A, B)
+ @test I != Union{}
+ @test I <: A
+ @test I <: B
+ # avoid stack overflow
+ J = typeintersect(A, C)
+ @test_broken J != Union{}
+end