Intersect: fix intersect_tuple for inputs with bad normalization. (J…

…uliaLang#49499)

src/subtype.c

@@ -3085,13 +3085,15 @@ static jl_value_t *intersect_varargs(jl_vararg_t *vmx, jl_vararg_t *vmy, ssize_t
  assert(e->Loffset == 0);
  e->Loffset = offset;
  jl_varbinding_t *xb = NULL, *yb = NULL;
- if (xp2 && jl_is_typevar(xp2)) {
+ if (xp2) {
+ assert(jl_is_typevar(xp2));
  xb = lookup(e, (jl_tvar_t*)xp2);
  if (xb) xb->intvalued = 1;
  if (!yp2)
  i2 = bound_var_below((jl_tvar_t*)xp2, xb, e, 0);
  }
- if (yp2 && jl_is_typevar(yp2)) {
+ if (yp2) {
+ assert(jl_is_typevar(yp2));
  yb = lookup(e, (jl_tvar_t*)yp2);
  if (yb) yb->intvalued = 1;
  if (!xp2)
@@ -3124,14 +3126,24 @@ static jl_value_t *intersect_varargs(jl_vararg_t *vmx, jl_vararg_t *vmy, ssize_t
 static jl_value_t *intersect_tuple(jl_datatype_t *xd, jl_datatype_t *yd, jl_stenv_t *e, int param)
 {
  size_t lx = jl_nparams(xd), ly = jl_nparams(yd);
+ size_t llx = lx, lly = ly;
  if (lx == 0 && ly == 0)
  return (jl_value_t*)yd;
- int vx=0, vy=0, vvx = (lx > 0 && jl_is_vararg(jl_tparam(xd, lx-1)));
- int vvy = (ly > 0 && jl_is_vararg(jl_tparam(yd, ly-1)));
- if (!vvx && !vvy && lx != ly)
- return jl_bottom_type;
+ int vx=0, vy=0;
+ jl_vararg_kind_t vvx = lx > 0 ? jl_vararg_kind(jl_tparam(xd, lx-1)) : JL_VARARG_NONE;
+ jl_vararg_kind_t vvy = ly > 0 ? jl_vararg_kind(jl_tparam(yd, ly-1)) : JL_VARARG_NONE;
+ if (vvx == JL_VARARG_INT)
+ llx += jl_unbox_long(jl_unwrap_vararg_num((jl_vararg_t *)jl_tparam(xd, lx-1))) - 1;
+ if (vvy == JL_VARARG_INT)
+ lly += jl_unbox_long(jl_unwrap_vararg_num((jl_vararg_t *)jl_tparam(yd, ly-1))) - 1;
+
+ if ((vvx == JL_VARARG_NONE || vvx == JL_VARARG_INT) &&
+ (vvy == JL_VARARG_NONE || vvy == JL_VARARG_INT)) {
+ if (llx != lly)
+ return jl_bottom_type;
+ }
 
- size_t np = lx > ly ? lx : ly;
+ size_t np = llx > lly ? llx : lly;
  jl_value_t *res = NULL;
  jl_svec_t *p = NULL;
  jl_value_t **params;
@@ -3150,20 +3162,20 @@ static jl_value_t *intersect_tuple(jl_datatype_t *xd, jl_datatype_t *yd, jl_sten
  int isx = 1, isy = 1; // try to reuse the object x or y as res whenever we can (e.g. when it is the supertype) instead of allocating a copy
  while (1) {
  vx = vy = 0;
- xi = i < lx ? jl_tparam(xd, i) : NULL;
- yi = j < ly ? jl_tparam(yd, j) : NULL;
+ xi = i < llx ? jl_tparam(xd, i < lx ? i : lx - 1) : NULL;
+ yi = j < lly ? jl_tparam(yd, j < ly ? j : ly - 1) : NULL;
  if (xi == NULL && yi == NULL) {
  assert(i == j && i == np);
  break;
  }
- if (xi && jl_is_vararg(xi)) vx = 1;
- if (yi && jl_is_vararg(yi)) vy = 1;
+ if (xi && jl_is_vararg(xi)) vx = vvx != JL_VARARG_INT;
+ if (yi && jl_is_vararg(yi)) vy = vvy != JL_VARARG_INT;
  if (xi == NULL || yi == NULL) {
- if (vx && intersect_vararg_length(xi, ly+1-lx, e, 0)) {
+ if (vx && intersect_vararg_length(xi, lly+1-llx, e, 0)) {
  np = j;
  p = NULL;
  }
- else if (vy && intersect_vararg_length(yi, lx+1-ly, e, 1)) {
+ else if (vy && intersect_vararg_length(yi, llx+1-lly, e, 1)) {
  np = i;
  p = NULL;
  }
@@ -3173,37 +3185,38 @@ static jl_value_t *intersect_tuple(jl_datatype_t *xd, jl_datatype_t *yd, jl_sten
  break;
  }
  jl_value_t *ii = NULL;
- if (vx && vy)
+ if (vx && vy) {
  ii = intersect_varargs((jl_vararg_t*)xi,
  (jl_vararg_t*)yi,
- ly - lx, // xi's offset: {A^n...,Vararg{T,N}} ∩ {Vararg{S,M}}
+ lly - llx, // xi's offset: {A^n...,Vararg{T,N}} ∩ {Vararg{S,M}}
  // {(A∩S)^n...,Vararg{T∩S,N}} plus N = M-n
  e,
  param);
+ }
  else {
- ii = intersect(vx ? jl_unwrap_vararg(xi) : xi,
- vy ? jl_unwrap_vararg(yi) : yi,
+ ii = intersect(jl_is_vararg(xi) ? jl_unwrap_vararg(xi) : xi,
+ jl_is_vararg(yi) ? jl_unwrap_vararg(yi) : yi,
  e,
  param == 0 ? 1 : param);
  }
  if (ii == jl_bottom_type) {
  if (vx && vy) {
  jl_varbinding_t *xb=NULL, *yb=NULL;
  jl_value_t *xlen = jl_unwrap_vararg_num(xi);
- if (xlen && jl_is_typevar(xlen))
-   xb = lookup(e, (jl_tvar_t*)xlen);
+ assert(xlen == NULL || jl_is_typevar(xlen));
+ if (xlen) xb = lookup(e, (jl_tvar_t*)xlen);
  jl_value_t *ylen = jl_unwrap_vararg_num(yi);
- if (ylen && jl_is_typevar(ylen))
-   yb = lookup(e, (jl_tvar_t*)ylen);
+ assert(ylen == NULL || jl_is_typevar(ylen));
+ if (ylen) yb = lookup(e, (jl_tvar_t*)ylen);
  int len = i > j ? i : j;
- if ((xb && jl_is_long(xb->lb) && lx-1+jl_unbox_long(xb->lb) != len) ||
- (yb && jl_is_long(yb->lb) && ly-1+jl_unbox_long(yb->lb) != len)) {
+ if ((xb && jl_is_long(xb->lb) && llx-1+jl_unbox_long(xb->lb) != len) ||
+ (yb && jl_is_long(yb->lb) && lly-1+jl_unbox_long(yb->lb) != len)) {
  res = jl_bottom_type;
  }
  else {
  assert(e->Loffset == 0);
- if (xb) set_var_to_const(xb, jl_box_long(len-lx+1), e, 0);
- if (yb) set_var_to_const(yb, jl_box_long(len-ly+1), e, 1);
+ if (xb) set_var_to_const(xb, jl_box_long(len-llx+1), e, 0);
+ if (yb) set_var_to_const(yb, jl_box_long(len-lly+1), e, 1);
  np = len;
  p = NULL;
  }
@@ -3221,8 +3234,8 @@ static jl_value_t *intersect_tuple(jl_datatype_t *xd, jl_datatype_t *yd, jl_sten
  params[i > j ? i : j] = ii;
  if (vx && vy)
  break;
- if (i < lx-1 || !vx) i++;
- if (j < ly-1 || !vy) j++;
+ if (!vx) i++;
+ if (!vy) j++;
  }
  // TODO: handle Vararg with explicit integer length parameter
  if (res == NULL) {

test/subtype.jl

@@ -2235,6 +2235,17 @@ let A = Pair{NTuple{N, Int}, Val{N}} where N,
  @testintersect A C C
 end
 
+# issue #49484
+let S = Tuple{Integer, U} where {II<:Array, U<:Tuple{Vararg{II, 1}}}
+ T = Tuple{Int, U} where {II<:Array, U<:Tuple{Vararg{II, 1}}}
+ @testintersect(S, Tuple{Int, U} where {U<:Tuple{Vararg{Any}}}, T)
+ @testintersect(S, Tuple{Int, U} where {N, U<:Tuple{Vararg{Any,N}}}, T)
+ @testintersect(S, Tuple{Int, U} where {U<:Tuple{Any,Vararg{Any}}}, T)
+ @testintersect(S, Tuple{Int, U} where {N, U<:Tuple{Any,Vararg{Any,N}}}, T)
+ @testintersect(S, Tuple{Int, U} where {U<:Tuple{Any,Any,Vararg{Any}}}, Union{})
+ @testintersect(S, Tuple{Int, U} where {N, U<:Tuple{Any,Any,Vararg{Any,N}}}, Union{})
+end
+
 # issue #43064
 let
  env_tuple(@nospecialize(x), @nospecialize(y)) = (intersection_env(x, y)[2]...,)