typeintersect: fuse unalias_unionall

And ensures all innervar get checked.

src/subtype.c

@@ -881,10 +881,20 @@ static jl_unionall_t *unalias_unionall(jl_unionall_t *u, jl_stenv_t *e)
  // in the environment, rename to get a fresh var.
  JL_GC_PUSH1(&u);
  while (btemp != NULL) {
- if (btemp->var == u->var ||
- // outer var can only refer to inner var if bounds changed
+ int aliased = btemp->var == u->var ||
+ // outer var can only refer to inner var if bounds changed (mainly for subtyping path)
  (btemp->lb != btemp->var->lb && jl_has_typevar(btemp->lb, u->var)) ||
- (btemp->ub != btemp->var->ub && jl_has_typevar(btemp->ub, u->var))) {
+ (btemp->ub != btemp->var->ub && jl_has_typevar(btemp->ub, u->var));
+ if (!aliased && btemp->innervars != NULL) {
+ for (size_t i = 0; i < jl_array_len(btemp->innervars); i++) {
+ jl_tvar_t *ivar = (jl_tvar_t*)jl_array_ptr_ref(btemp->innervars, i);
+ if (ivar == u->var) {
+ aliased = 1;
+ break;
+ }
+ }
+ }
+ if (aliased) {
  u = jl_rename_unionall(u);
  break;
  }
@@ -3056,23 +3066,16 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind
 static jl_value_t *intersect_unionall_(jl_value_t *t, jl_unionall_t *u, jl_stenv_t *e, int8_t R, int param, jl_varbinding_t *vb)
 {
  jl_varbinding_t *btemp = e->vars;
- // if the var for this unionall (based on identity) already appears somewhere
- // in the environment, rename to get a fresh var.
- // TODO: might need to look inside types in btemp->lb and btemp->ub
  int envsize = 0;
  while (btemp != NULL) {
  envsize++;
  if (envsize > 120) {
  vb->limited = 1;
  return t;
  }
- if (btemp->var == u->var || btemp->lb == (jl_value_t*)u->var ||
- btemp->ub == (jl_value_t*)u->var) {
- u = jl_rename_unionall(u);
- break;
- }
  btemp = btemp->prev;
  }
+ u = unalias_unionall(u, e);
  JL_GC_PUSH1(&u);
  vb->var = u->var;
  e->vars = vb;

test/subtype.jl

@@ -1216,7 +1216,8 @@ let a = Tuple{T1,T1} where T1,
 end
 let a = Val{Tuple{T1,T1}} where T1,
  b = Val{Tuple{Val{S2},S6}} where S2 where S6
- @testintersect(a, b, Val{Tuple{Val{T},Val{T}}} where T)
+ # @testintersect(a, b, Val{Tuple{Val{T},Val{T}}} where T)
+ @test_broken !Base.has_free_typevars(typeintersect(b, a))
 end
 let a = Tuple{Float64,T3,T4} where T4 where T3,
  b = Tuple{S2,Tuple{S3},S3} where S2 where S3
@@ -2572,6 +2573,32 @@ let a = Tuple{Union{Nothing, Type{Pair{T1}} where T1}}
  @test !Base.has_free_typevars(typeintersect(a, b))
 end
 
+#issue 53366
+let Y = Tuple{Val{T}, Val{Val{T}}} where T
+ A = Val{Val{T}} where T
+ T = TypeVar(:T, UnionAll(A.var, Val{A.var}))
+ B = UnionAll(T, Val{T})
+ X = Tuple{A, B}
+ @testintersect(X, Y, !Union{})
+end
+
+#issue 53621 (requires assertions enabled)
+abstract type A53621{T, R, C, U} <: AbstractSet{Union{C, U}} end
+struct T53621{T, R<:Real, C, U} <: A53621{T, R, C, U} end
+let
+ U = TypeVar(:U)
+ C = TypeVar(:C)
+ T = TypeVar(:T)
+ R = TypeVar(:R)
+ CC = TypeVar(:CC, Union{C, U})
+ UU = TypeVar(:UU, Union{C, U})
+ S1 = UnionAll(T, UnionAll(R, Type{UnionAll(C, UnionAll(U, T53621{T, R, C, U}))}))
+ S2 = UnionAll(C, UnionAll(U, UnionAll(CC, UnionAll(UU, UnionAll(T, UnionAll(R, T53621{T, R, CC, UU}))))))
+ S = Tuple{S1, S2}
+ T = Tuple{Type{T53621{T, R}}, AbstractSet{T}} where {T, R}
+ @testintersect(S, T, !Union{})
+end
+
 #issue 53371
 struct T53371{A,B,C,D,E} end
 S53371{A} = Union{Int, <:A}