fix an optimizer bug in invoke with non-constant functions (#26301)

In this case inlining was able to convert an `invoke` call to an
`:invoke` Expr, but we had not inferred the target method instance,
which led us to hit the fallback path in `jl_invoke`, which does not
support this case yet.

base/compiler/tfuncs.jl

@@ -729,12 +729,15 @@ function apply_type_tfunc(@nospecialize(headtypetype), @nospecialize args...)
 end
 add_tfunc(apply_type, 1, INT_INF, apply_type_tfunc, 10)
 
-function invoke_tfunc(@nospecialize(f), @nospecialize(types), @nospecialize(argtype), sv::InferenceState)
+function invoke_tfunc(@nospecialize(ft), @nospecialize(types), @nospecialize(argtype), sv::InferenceState)
+ dt = ccall(:jl_argument_datatype, Any, (Any,), ft)
+ if dt === nothing || !isdefined(dt.name, :mt)
+ return Any
+ end
  argtype = typeintersect(types, limit_tuple_type(argtype, sv.params))
  if argtype === Bottom
  return Bottom
  end
- ft = Core.Typeof(f)
  types = rewrap_unionall(Tuple{ft, unwrap_unionall(types).parameters...}, types)
  argtype = Tuple{ft, argtype.parameters...}
  entry = ccall(:jl_gf_invoke_lookup, Any, (Any, UInt), types, sv.params.world)
@@ -816,8 +819,12 @@ function builtin_tfunction(@nospecialize(f), argtypes::Array{Any,1},
  end
  return Expr
  elseif f === invoke
- if length(argtypes) > 1 && isa(argtypes[1], Const) && sv !== nothing
- af = argtypes[1].val
+ if length(argtypes) > 1 && sv !== nothing && (isa(argtypes[1], Const) || isa(argtypes[1], Type))
+ if isa(argtypes[1], Const)
+ ft = Core.Typeof(argtypes[1].val)
+ else
+ ft = argtypes[1]
+ end
  sig = argtypes[2]
  if isa(sig, Const)
  sigty = sig.val
@@ -827,7 +834,7 @@ function builtin_tfunction(@nospecialize(f), argtypes::Array{Any,1},
  sigty = nothing
  end
  if isa(sigty, Type) && !has_free_typevars(sigty) && sigty <: Tuple
- return invoke_tfunc(af, sigty, argtypes_to_type(argtypes[3:end]), sv)
+ return invoke_tfunc(ft, sigty, argtypes_to_type(argtypes[3:end]), sv)
  end
  end
  return Any

src/gf.c

@@ -48,6 +48,10 @@ JL_DLLEXPORT jl_value_t *jl_invoke(jl_method_instance_t *meth, jl_value_t **args
  // since it can go through the unrolled caches for this world
  // and if inference is successful, this meth would get updated anyways,
  // and we'll get the fast path here next time
+
+ // TODO: if `meth` came from an `invoke` call, we should make sure
+ // meth->def is called instead of doing normal dispatch.
+
  return jl_apply(args, nargs);
  }
 }

test/core.jl

@@ -4852,6 +4852,23 @@ f18095(::Number, ::Int) = 0x12
 @test_throws MethodError invoke(f18095, Tuple{Int, Any}, 1, 2)
 @test invoke(f18095, Tuple{Int, Real}, 1, 2) === 0x21
 
+# `invoke` with non-constant function
+struct CassetteLikeWrapper{F}
+ x
+ f::F
+end
+(foo::CassetteLikeWrapper)(args...) = foo.f(args...)
+(foo::CassetteLikeWrapper)(x) = invoke(foo, Tuple{Vararg{Any}}, x)
+@test CassetteLikeWrapper(1,-)(2) == -2
+
+f26301(x) = 1
+f26301(x::Int) = 2
+function g26301()
+ f = Any[f26301][1]
+ invoke(f, Tuple{Any}, 0)
+end
+@test g26301() == 1
+
 # issue #10981, long argument lists
 let a = fill(["sdf"], 2*10^6), temp_vcat(x...) = vcat(x...)
  # we introduce a new function `temp_vcat` to make sure there is no existing