fix obvious_subtype bug with egal objects (JuliaLang#45771)

When egal objects contain identical typevars with different environments, the resulting subtyping might not be so obvious. Fix JuliaLang#45703
pcjentsch · Aug 18, 2022 · a714bb5 · a714bb5
1 parent aa952ae
commit a714bb5
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/src/subtype.c b/src/subtype.c
@@ -1542,8 +1542,15 @@ static int obvious_subtype(jl_value_t *x, jl_value_t *y, jl_value_t *y0, int *su
  *subtype = 1;
  return 1;
  }
- if (jl_is_unionall(x))
- x = jl_unwrap_unionall(x);
+ while (jl_is_unionall(x)) {
+ if (!jl_is_unionall(y)) {
+ if (obvious_subtype(jl_unwrap_unionall(x), y, y0, subtype) && !*subtype)
+ return 1;
+ return 0;
+ }
+ x = ((jl_unionall_t*)x)->body;
+ y = ((jl_unionall_t*)y)->body;
+ }
  if (jl_is_unionall(y))
  y = jl_unwrap_unionall(y);
  if (x == (jl_value_t*)jl_typeofbottom_type->super)

diff --git a/test/subtype.jl b/test/subtype.jl
@@ -1987,3 +1987,12 @@ let A = Tuple{typeof(identity), Type{Union{}}},
  B = Tuple{typeof(identity), typeof(Union{})}
  @test A == B && (Base.isdispatchtuple(A) == Base.isdispatchtuple(B))
 end
+
+# issue #45703
+# requires assertions enabled (to catch discrepancy in obvious_subtype)
+let T = TypeVar(:T, Real),
+ V = TypeVar(:V, AbstractVector{T}),
+ S = Type{Pair{T, V}}
+ @test !(UnionAll(T, UnionAll(V, UnionAll(T, Type{Pair{T, V}}))) <: UnionAll(T, UnionAll(V, Type{Pair{T, V}})))
+ @test !(UnionAll(T, UnionAll(V, UnionAll(T, S))) <: UnionAll(T, UnionAll(V, S)))
+end