abi: thread first argument through separately (#31984)

A bit of history: Before the addition of function types, a call to a function would always use the API `(jl_value_t *f, jl_value_t *argv[], int nargv)`, regardless of whether it was a generic call or a closure or an invoke or interpreted. Over time since then, that become one of several calling conventions, each just a minor variation away: `(jl_value_t *argv[], int nargv)`, `(void *context, jl_value_t *argv[], int nargv)`, `(jl_value_t *f, jl_value_t *argv[], int nargs)`,  `(jl_value_t **context, jl_value_t *f, jl_value_t *argv[], int nargv)`, and so on. That was fairly nice to getting stuff ported over, since it provided a simpler transition, but led to needing an increasing number of transformational methods to just shift arguments around (such as `jl_apply_2va`, which gets removed here). And that was starting to feel tricky to me to manage the possible N-to-N conversions that may be needed between each of them, and to optimize all of those.

This PR aims to cut back down again on the amount of variety in our primary (non internal/specsig) ABI by moving the varying 'context' to the end and usually keeping the first (function) argument separate. This gives the basic call signature form: `(jl_value_t *func, jl_value_t *argv[], int nargv, void *optional_context)` (where the last parameter can range over a few possible types depending on the target, or may be `undef` and dropped when it is known that the callee will ignore the value). This lets us switch callee target type (e.g. for incremental compilation and linking) simply by adding / dropping / replacing / ignoring that trailing parameter, and not needing to potentially spill everything to the stack (e.g. letting us remove jl_apply_2va).

This doesn't alter `jl_apply()` / `jl_call` / etc., so external consumers should be generally unaffected.

src/ast.c

@@ -1030,13 +1030,14 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule
  jl_value_t *result;
  JL_TRY {
  margs[0] = jl_toplevel_eval(*ctx, margs[0]);
- jl_method_instance_t *mfunc = jl_method_lookup(jl_gf_mtable(margs[0]), margs, nargs, 1, world);
+ jl_method_instance_t *mfunc = jl_method_lookup(margs, nargs, 1, world);
+ JL_GC_PROMISE_ROOTED(mfunc);
  if (mfunc == NULL) {
- jl_method_error((jl_function_t*)margs[0], margs, nargs, world);
+ jl_method_error(margs[0], &margs[1], nargs, world);
  // unreachable
  }
  *ctx = mfunc->def.method->module;
- result = jl_invoke(mfunc, margs, nargs);
+ result = jl_invoke(margs[0], &margs[1], nargs - 1, mfunc);
  }
  JL_CATCH {
  if (jl_loaderror_type == NULL) {

src/builtins.c

@@ -671,18 +671,19 @@ JL_CALLABLE(jl_f__apply_latest)
 JL_CALLABLE(jl_f_tuple)
 {
  size_t i;
- if (nargs == 0) return (jl_value_t*)jl_emptytuple;
+ if (nargs == 0)
+ return (jl_value_t*)jl_emptytuple;
  jl_datatype_t *tt;
- if (nargs < jl_page_size/sizeof(jl_value_t*)) {
- jl_value_t **types = (jl_value_t**)alloca(nargs*sizeof(jl_value_t*));
- for(i=0; i < nargs; i++)
+ if (nargs < jl_page_size / sizeof(jl_value_t*)) {
+ jl_value_t **types = (jl_value_t**)alloca(nargs * sizeof(jl_value_t*));
+ for (i = 0; i < nargs; i++)
  types[i] = jl_typeof(args[i]);
  tt = jl_inst_concrete_tupletype_v(types, nargs);
  }
  else {
  jl_svec_t *types = jl_alloc_svec_uninit(nargs);
  JL_GC_PUSH1(&types);
- for(i=0; i < nargs; i++)
+ for (i = 0; i < nargs; i++)
  jl_svecset(types, i, jl_typeof(args[i]));
  tt = jl_inst_concrete_tupletype(types);
  JL_GC_POP();
@@ -955,7 +956,7 @@ JL_CALLABLE(jl_f_applicable)
 {
  JL_NARGSV(applicable, 1);
  size_t world = jl_get_ptls_states()->world_age;
- return jl_method_lookup(jl_gf_mtable(args[0]), args, nargs, 1, world) != NULL ?
+ return jl_method_lookup(args, nargs, 1, world) != NULL ?
  jl_true : jl_false;
 }
 
@@ -966,10 +967,9 @@ JL_CALLABLE(jl_f_invoke)
  JL_GC_PUSH1(&argtypes);
  if (!jl_is_tuple_type(jl_unwrap_unionall(args[1])))
  jl_type_error("invoke", (jl_value_t*)jl_anytuple_type_type, args[1]);
- if (!jl_tuple_isa(&args[2], nargs-2, (jl_datatype_t*)argtypes))
+ if (!jl_tuple_isa(&args[2], nargs - 2, (jl_datatype_t*)argtypes))
  jl_error("invoke: argument type error");
- args[1] = args[0]; // move function directly in front of arguments
- jl_value_t *res = jl_gf_invoke(argtypes, &args[1], nargs-1);
+ jl_value_t *res = jl_gf_invoke(argtypes, args[0], &args[2], nargs - 1);
  JL_GC_POP();
  return res;
 }

src/codegen.cpp

@@ -280,7 +280,6 @@ static Function *jltuple_func;
 static Function *jlnsvec_func;
 static Function *jlapplygeneric_func;
 static Function *jlinvoke_func;
-static Function *jlapply2va_func;
 static Function *jlgetfield_func;
 static Function *jlmethod_func;
 static Function *jlgenericfunction_func;
@@ -336,10 +335,6 @@ static Function *gc_preserve_end_func;
 static Function *except_enter_func;
 static Function *pointer_from_objref_func;
 
-static std::vector<Type *> two_pvalue_llvmt;
-static std::vector<Type *> three_pvalue_llvmt;
-static std::vector<Type *> four_pvalue_llvmt;
-
 static std::map<jl_fptr_args_t, Function*> builtin_func_map;
 
 // --- code generation ---
@@ -552,8 +547,8 @@ static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i);
 static Value *emit_condition(jl_codectx_t &ctx, const jl_cgval_t &condV, const std::string &msg);
 static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0);
 static void CreateTrap(IRBuilder<> &irbuilder);
-static Value *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF,
- jl_cgval_t *args, size_t nargs);
+static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF,
+  jl_cgval_t *args, size_t nargs, CallingConv::ID cc);
 
 static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p);
 static GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G);
@@ -2190,7 +2185,7 @@ static jl_cgval_t emit_getfield(jl_codectx_t &ctx, const jl_cgval_t &strct, jl_s
  strct,
  mark_julia_const((jl_value_t*)name)
  };
- Value *result = emit_jlcall(ctx, jlgetfield_func, maybe_decay_untracked(V_null), myargs_array, 2);
+ Value *result = emit_jlcall(ctx, jlgetfield_func, maybe_decay_untracked(V_null), myargs_array, 2, JLCALL_F_CC);
  return mark_julia_type(ctx, result, true, jl_any_type);
 }
 
@@ -2503,7 +2498,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
  nva = ctx.builder.CreateTrunc(nva, T_int32);
 #endif
  Value *theArgs = ctx.builder.CreateInBoundsGEP(ctx.argArray, ConstantInt::get(T_size, ctx.nReqArgs));
- Value *r = ctx.builder.CreateCall(prepare_call(jlapply2va_func), { theF, theArgs, nva });
+ Value *r = ctx.builder.CreateCall(prepare_call(jlapplygeneric_func), { theF, theArgs, nva });
  *ret = mark_julia_type(ctx, r, true, jl_any_type);
  return true;
  }
@@ -3029,30 +3024,27 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
  return false;
 }
 
-static Value *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF,
- jl_cgval_t *argv, size_t nargs)
+static CallInst *emit_jlcall(jl_codectx_t &ctx, Value *theFptr, Value *theF,
+  jl_cgval_t *argv, size_t nargs, CallingConv::ID cc)
 {
  // emit arguments
  SmallVector<Value*, 3> theArgs;
- if (theF)
+ SmallVector<Type*, 3> argsT;
+ if (theF) {
  theArgs.push_back(theF);
+ argsT.push_back(T_prjlvalue);
+ }
  for (size_t i = 0; i < nargs; i++) {
  Value *arg = maybe_decay_untracked(boxed(ctx, argv[i]));
  theArgs.push_back(arg);
- }
- SmallVector<Type *, 3> argsT;
- for (size_t i = 0; i < nargs + (theF != nullptr); i++) {
  argsT.push_back(T_prjlvalue);
  }
  FunctionType *FTy = FunctionType::get(T_prjlvalue, argsT, false);
  CallInst *result = ctx.builder.CreateCall(FTy,
  ctx.builder.CreateBitCast(prepare_call(theFptr), FTy->getPointerTo()),
  theArgs);
  add_return_attr(result, Attribute::NonNull);
- if (theF)
- result->setCallingConv(JLCALL_F_CC);
- else
- result->setCallingConv(JLCALL_CC);
+ result->setCallingConv(cc);
  return result;
 }
 
@@ -3181,7 +3173,7 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, StringRef specFunct
  add_return_attr(F, Attribute::NonNull);
  F->addFnAttr(Thunk);
  }
- Value *ret = emit_jlcall(ctx, theFptr, boxed(ctx, argv[0]), &argv[1], nargs - 1);
+ Value *ret = emit_jlcall(ctx, theFptr, nullptr, argv, nargs, JLCALL_F_CC);
  return mark_julia_type(ctx, ret, true, inferred_retty);
 }
 
@@ -3227,14 +3219,8 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt)
  }
  }
  if (!handled) {
- result = mark_julia_type(ctx,
- emit_jlcall(
- ctx,
- prepare_call(jlinvoke_func),
- boxed(ctx, lival),
- argv, nargs),
- true,
- rt);
+ Value *r = emit_jlcall(ctx, prepare_call(jlinvoke_func), boxed(ctx, lival), argv, nargs, JLCALL_F2_CC);
+ result = mark_julia_type(ctx, r, true, rt);
  }
  if (result.typ == jl_bottom_type)
  CreateTrap(ctx.builder);
@@ -3274,13 +3260,13 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt)
  std::map<jl_fptr_args_t, Function*>::iterator it = builtin_func_map.find(jl_get_builtin_fptr(f.constant));
  if (it != builtin_func_map.end()) {
  Value *theFptr = it->second;
- Value *ret = emit_jlcall(ctx, theFptr, maybe_decay_untracked(V_null), &argv[1], nargs - 1);
+ Value *ret = emit_jlcall(ctx, theFptr, maybe_decay_untracked(V_null), &argv[1], nargs - 1, JLCALL_F_CC);
  return mark_julia_type(ctx, ret, true, rt);
  }
  }
 
  // emit function and arguments
- Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs);
+ Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, JLCALL_F_CC);
  return mark_julia_type(ctx, callval, true, rt);
 }
 
@@ -4145,8 +4131,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval)
  assert(nargs <= jl_datatype_nfields(jl_tparam0(ty)) + 1);
  return emit_new_struct(ctx, jl_tparam0(ty), nargs - 1, &argv[1]);
  }
- Value *typ = boxed(ctx, argv[0]);
- Value *val = emit_jlcall(ctx, jlnew_func, typ, &argv[1], nargs - 1);
+ Value *val = emit_jlcall(ctx, jlnew_func, nullptr, argv, nargs, JLCALL_F_CC);
  // temporarily mark as `Any`, expecting `emit_ssaval_assign` to update
  // it to the inferred type.
  return mark_julia_type(ctx, val, true, (jl_value_t*)jl_any_type);
@@ -4343,7 +4328,7 @@ static void emit_cfunc_invalidate(
  }
  }
  assert(AI == gf_thunk->arg_end());
- Value *gf_ret = emit_jlcall(ctx, jlapplygeneric_func, NULL, myargs, nargs);
+ Value *gf_ret = emit_jlcall(ctx, jlapplygeneric_func, nullptr, myargs, nargs, JLCALL_F_CC);
  jl_cgval_t gf_retbox = mark_julia_type(ctx, gf_ret, true, jl_any_type);
  jl_value_t *astrt = codeinst->rettype;
  if (cc != jl_returninfo_t::Boxed) {
@@ -4715,11 +4700,11 @@ static Function* gen_cfun_wrapper(
  // for jlcall, we need to pass the function object even if it is a ghost.
  Value *theF = boxed(ctx, inputargs[0]);
  assert(theF);
- ret_jlcall = emit_jlcall(ctx, theFptr, theF, &inputargs[1], nargs);
+ ret_jlcall = emit_jlcall(ctx, theFptr, theF, &inputargs[1], nargs, JLCALL_F_CC);
  ctx.builder.CreateBr(b_after);
  ctx.builder.SetInsertPoint(b_generic);
  }
- Value *ret = emit_jlcall(ctx, prepare_call(jlapplygeneric_func), NULL, inputargs, nargs + 1);
+ Value *ret = emit_jlcall(ctx, prepare_call(jlapplygeneric_func), NULL, inputargs, nargs + 1, JLCALL_F_CC);
  if (age_ok) {
  ctx.builder.CreateBr(b_after);
  ctx.builder.SetInsertPoint(b_after);
@@ -5168,9 +5153,10 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret
  w->addFnAttr("no-frame-pointer-elim", "true");
 #endif
  Function::arg_iterator AI = w->arg_begin();
- Value *methodArg = &*AI++; (void)methodArg;
+ Value *funcArg = &*AI++;
  Value *argArray = &*AI++;
- Value *argCount = &*AI++; (void)argCount;
+ Value *argCount = &*AI++; (void)argCount; // unused
+ //Value *mfunc = &*AI++; (void)mfunc; // unused
  assert(AI == w->arg_end());
 
  jl_codectx_t ctx(jl_LLVMContext);
@@ -5214,9 +5200,15 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret
  Type *lty = julia_type_to_llvm(ty, &isboxed);
  if (lty != NULL && type_is_ghost(lty))
  continue;
- Value *argPtr = ctx.builder.CreateInBoundsGEP(argArray, ConstantInt::get(T_size, i));
- Value *theArg = maybe_mark_load_dereferenceable(ctx.builder.CreateLoad(argPtr), false, ty);
- if (lty != NULL && !isboxed) {
+ Value *theArg;
+ if (i == 0) {
+ theArg = funcArg;
+ }
+ else {
+ Value *argPtr = ctx.builder.CreateInBoundsGEP(argArray, ConstantInt::get(T_size, i - 1));
+ theArg = maybe_mark_load_dereferenceable(ctx.builder.CreateLoad(argPtr), false, ty);
+ }
+ if (!isboxed) {
  theArg = decay_derived(emit_bitcast(ctx, theArg, PointerType::get(lty, 0)));
  if (!lty->isAggregateType()) // keep "aggregate" type values in place as pointers
  theArg = ctx.builder.CreateAlignedLoad(theArg, julia_alignment(ty));
@@ -5737,17 +5729,16 @@ static std::unique_ptr<Module> emit_function(
  Value *fArg=NULL, *argArray=NULL, *pargArray=NULL, *argCount=NULL;
  if (!specsig) {
  Function::arg_iterator AI = f->arg_begin();
- if (needsparams) {
- ctx.spvals_ptr = &*AI;
- ++AI;
- }
  fArg = &*AI++;
  argArray = &*AI++;
  pargArray = ctx.builder.CreateAlloca(argArray->getType());
  ctx.builder.CreateStore(argArray, pargArray, true/*volatile store to prevent removal of this alloca*/);
  argCount = &*AI++;
  ctx.argArray = argArray;
  ctx.argCount = argCount;
+ if (needsparams) {
+ ctx.spvals_ptr = &*AI++;
+ }
  }
 
  /*
@@ -6009,8 +6000,9 @@ static std::unique_ptr<Module> emit_function(
  // FIXME: this may assert since the type of vi might not be isbits here
  emit_varinfo_assign(ctx, vi, tuple);
  } else {
- jl_cgval_t tuple = mark_julia_type(ctx, emit_jlcall(ctx, prepare_call(jltuple_func), maybe_decay_untracked(V_null),
- vargs, ctx.nvargs), true, vi.value.typ);
+ Value *vtpl = emit_jlcall(ctx, prepare_call(jltuple_func), maybe_decay_untracked(V_null),
+ vargs, ctx.nvargs, JLCALL_F_CC);
+ jl_cgval_t tuple = mark_julia_type(ctx, vtpl, true, vi.value.typ);
  emit_varinfo_assign(ctx, vi, tuple);
  }
  }
@@ -7029,28 +7021,18 @@ static void init_julia_llvm_env(Module *m)
  T_prjlvalue = PointerType::get(T_jlvalue, AddressSpace::Tracked);
  T_ppjlvalue = PointerType::get(T_pjlvalue, 0);
  T_pprjlvalue = PointerType::get(T_prjlvalue, 0);
- two_pvalue_llvmt.push_back(T_pjlvalue);
- two_pvalue_llvmt.push_back(T_pjlvalue);
- three_pvalue_llvmt.push_back(T_pjlvalue);
- three_pvalue_llvmt.push_back(T_pjlvalue);
- three_pvalue_llvmt.push_back(T_pjlvalue);
- four_pvalue_llvmt.push_back(T_pjlvalue);
- four_pvalue_llvmt.push_back(T_pjlvalue);
- four_pvalue_llvmt.push_back(T_pjlvalue);
- four_pvalue_llvmt.push_back(T_pjlvalue);
  V_null = Constant::getNullValue(T_pjlvalue);
  jl_init_jit(T_pjlvalue);
 
  std::vector<Type*> ftargs(0);
- ftargs.push_back(T_pprjlvalue); // linfo->sparam_vals
  ftargs.push_back(T_prjlvalue); // function
  ftargs.push_back(T_pprjlvalue); // args[]
  ftargs.push_back(T_int32); // nargs
- jl_func_sig_sparams = FunctionType::get(T_prjlvalue, ftargs, false);
- assert(jl_func_sig_sparams != NULL);
- ftargs.erase(ftargs.begin()); // drop linfo->sparams_vals argument
  jl_func_sig = FunctionType::get(T_prjlvalue, ftargs, false);
  assert(jl_func_sig != NULL);
+ ftargs.push_back(T_pprjlvalue); // linfo->sparam_vals
+ jl_func_sig_sparams = FunctionType::get(T_prjlvalue, ftargs, false);
+ assert(jl_func_sig_sparams != NULL);
 
  Type *vaelts[] = {PointerType::get(T_int8, AddressSpace::Loaded)
 #ifdef STORE_ARRAY_LEN
@@ -7288,26 +7270,24 @@ static void init_julia_llvm_env(Module *m)
  jltuple_func = builtin_func_map[jl_f_tuple];
  jlgetfield_func = builtin_func_map[jl_f_getfield];
 
- jlapply2va_func = jlcall_func_to_llvm("jl_apply_2va", &jl_apply_2va, m);
-
- std::vector<Type *> agargs(0);
- agargs.push_back(T_pprjlvalue);
- agargs.push_back(T_uint32);
- jlapplygeneric_func = Function::Create(FunctionType::get(T_prjlvalue, agargs, false),
+ jlapplygeneric_func = Function::Create(jl_func_sig,
  Function::ExternalLinkage,
  "jl_apply_generic", m);
  add_return_attr(jlapplygeneric_func, Attribute::NonNull);
  jlapplygeneric_func->addFnAttr(Thunk);
  add_named_global(jlapplygeneric_func, &jl_apply_generic);
 
- std::vector<Type *> invokeargs(0);
+ std::vector<Type*> invokeargs(0);
  invokeargs.push_back(T_prjlvalue);
  invokeargs.push_back(T_pprjlvalue);
  invokeargs.push_back(T_uint32);
+ invokeargs.push_back(T_prjlvalue);
  jlinvoke_func = Function::Create(FunctionType::get(T_prjlvalue, invokeargs, false),
  Function::ExternalLinkage,
  "jl_invoke", m);
  add_return_attr(jlinvoke_func, Attribute::NonNull);
+ jlinvoke_func->addAttribute(2, Attribute::ReadOnly);
+ jlinvoke_func->addAttribute(2, Attribute::NoCapture);
  add_named_global(jlinvoke_func, &jl_invoke);
 
  std::vector<Type *> exp_args(0);

src/codegen_shared.h

@@ -13,8 +13,9 @@ enum AddressSpace {
  LastSpecial = Loaded,
 };
 
-#define JLCALL_CC (CallingConv::ID)36
-#define JLCALL_F_CC (CallingConv::ID)37
+// JLCALL with API arguments ([extra], arg0, arg1, arg2, ...) has the following ABI calling conventions defined:
+#define JLCALL_F_CC (CallingConv::ID)37 // (jl_value_t *arg0, jl_value_t **argv, uint32_t nargv)
+#define JLCALL_F2_CC (CallingConv::ID)38 // (jl_value_t *arg0, jl_value_t **argv, uint32_t nargv, jl_value_t *extra)
 
 static inline void llvm_dump(llvm::Value *v)
 {