gf lookup: avoid creating unintended specializations and cache misses (…

…JuliaLang#36704)

The typemap code was forgetting to handle the case where the user needed
to look for something complicated (like AbstractVec{Float64}) in the
method table cache (such as for ==).

base/compiler/typelattice.jl

@@ -84,8 +84,10 @@ struct NotFound end
 
 const NOT_FOUND = NotFound()
 
-const CompilerTypes = Union{Type, MaybeUndef, Const, Conditional, NotFound, PartialStruct}
+const CompilerTypes = Union{MaybeUndef, Const, Conditional, NotFound, PartialStruct}
 ==(x::CompilerTypes, y::CompilerTypes) = x === y
+==(x::Type, y::CompilerTypes) = false
+==(x::CompilerTypes, y::Type) = false
 
 #################
 # lattice logic #

base/show.jl

@@ -2492,11 +2492,11 @@ type, indicating that any recursed calls are not at the top level.
 Printing the parent as `::Array{Float64,3}` is the fallback (non-toplevel)
 behavior, because no specialized method for `Array` has been defined.
 """
-function showarg(io::IO, ::Type{T}, toplevel) where {T}
+function showarg(io::IO, T::Type, toplevel)
  toplevel || print(io, "::")
  print(io, "Type{", T, "}")
 end
-function showarg(io::IO, x, toplevel)
+function showarg(io::IO, @nospecialize(x), toplevel)
  toplevel || print(io, "::")
  print(io, typeof(x))
 end

src/typemap.c

@@ -967,21 +967,34 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v
  jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world);
  if (ml) return ml;
  }
- if (jl_is_kind(ty) && cache->tname != (jl_array_t*)jl_an_empty_vec_any) {
+ jl_array_t *tname = cache->tname;
+ if (jl_is_kind(ty) && tname != (jl_array_t*)jl_an_empty_vec_any) {
  jl_value_t *name = jl_type_extract_name(a1);
  if (name) {
  if (ty != (jl_value_t*)jl_datatype_type)
  a1 = jl_unwrap_unionall(((jl_typename_t*)name)->wrapper);
  while (1) {
  jl_typemap_t *ml_or_cache = mtcache_hash_lookup(
- cache->tname, (jl_value_t*)((jl_datatype_t*)a1)->name);
+ tname, (jl_value_t*)((jl_datatype_t*)a1)->name);
  jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world);
  if (ml) return ml;
  if (a1 == (jl_value_t*)jl_any_type)
  break;
  a1 = (jl_value_t*)((jl_datatype_t*)a1)->super;
  }
  }
+ else {
+ // couldn't figure out unique `name` initial point, so must scan all for matches
+ size_t i, l = jl_array_len(tname);
+ jl_typemap_t **data = (jl_typemap_t **)jl_array_ptr_data(tname);
+ for (i = 1; i < l; i += 2) {
+ jl_typemap_t *ml_or_cache = data[i];
+ if (ml_or_cache == NULL || ml_or_cache == jl_nothing)
+ continue;
+ jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs + 1, world);
+ if (ml) return ml;
+ }
+ }
  }
  if (cache->name1 != (jl_array_t*)jl_an_empty_vec_any) {
  while (1) {

test/subarray.jl

@@ -36,7 +36,7 @@ _Agen(A, i1, i2, i3, i4, i5, i6) = [A[j1,j2,j3,j4,j5,j6] for j1 in i1, j2 in i2,
 
 function replace_colon(A::AbstractArray, I)
  Iout = Vector{Any}(undef, length(I))
- I == (:,) && return (1:length(A),)
+ I === (:,) && return (1:length(A),)
  for d = 1:length(I)
  Iout[d] = isa(I[d], Colon) ? (1:size(A,d)) : I[d]
  end