increment jlcall_api to make 0 invalid

this lets us avoid needing atomic operations to order the reads of fptr and jlcall_api
since both will be null-initialized, set-once while the codegen lock is held

src/alloc.c

@@ -456,15 +456,15 @@ STATIC_INLINE jl_value_t *jl_call_staged(jl_svec_t *sparam_vals, jl_lambda_info_
  jl_generic_fptr_t fptr;
  fptr.fptr = generator->fptr;
  fptr.jlcall_api = generator->jlcall_api;
- if (__unlikely(fptr.fptr == NULL)) {
+ if (__unlikely(fptr.fptr == NULL || fptr.jlcall_api == 0)) {
  void *F = jl_compile_linfo(generator, (jl_source_info_t*)generator->inferred).functionObject;
  fptr = jl_generate_fptr(generator, F);
  }
  assert(jl_svec_len(generator->def->sparam_syms) == jl_svec_len(sparam_vals));
- if (fptr.jlcall_api == 0)
- return fptr.fptr0(args[0], &args[1], nargs-1);
- else if (fptr.jlcall_api == 1)
- return fptr.fptr1(sparam_vals, args[0], &args[1], nargs-1);
+ if (fptr.jlcall_api == 1)
+ return fptr.fptr1(args[0], &args[1], nargs-1);
+ else if (fptr.jlcall_api == 3)
+ return fptr.fptr3(sparam_vals, args[0], &args[1], nargs-1);
  else
  abort(); // shouldn't have inferred any other calling convention
 }

src/anticodegen.c

@@ -47,5 +47,5 @@ jl_value_t *jl_interpret_call(jl_lambda_info_t *lam, jl_value_t **args, uint32_t
 void jl_generate_fptr(jl_lambda_info_t *li)
 {
  li->fptr = (jl_fptr_t)&jl_interpret_call;
- li->jlcall_api = 3;
+ li->jlcall_api = 4;
 }

src/codegen.cpp

@@ -203,7 +203,7 @@ static Type *T_jlvalue;
 static Type *T_pjlvalue;
 static Type *T_ppjlvalue;
 static Type *jl_parray_llvmt;
-FunctionType *jl_func_sig;
+static FunctionType *jl_func_sig;
 static FunctionType *jl_func_sig_sparams;
 static Type *T_pvoidfunc;
 
@@ -279,6 +279,18 @@ DICompositeType jl_di_func_null_sig;
 #endif
 #endif
 
+
+extern "C"
+int32_t jl_jlcall_api(const void *function)
+{
+ // give the function an index in the constant lookup table
+ if (function == NULL)
+ return 0;
+ Function *F = (Function*)function;
+ return (F->getFunctionType() == jl_func_sig ? 1 : 3);
+}
+
+
 // constants
 static Value *V_null;
 static Type *NoopType;
@@ -1010,33 +1022,33 @@ jl_generic_fptr_t jl_generate_fptr(jl_lambda_info_t *li, void *_F)
  jl_generic_fptr_t fptr;
  fptr.fptr = li->fptr;
  fptr.jlcall_api = li->jlcall_api;
- if (fptr.fptr) {
+ if (fptr.fptr && fptr.jlcall_api) {
  return fptr;
  }
  fptr.fptr = li->unspecialized_ducttape;
- fptr.jlcall_api = 0;
+ fptr.jlcall_api = 1;
  if (!li->inferred && fptr.fptr) {
  return fptr;
  }
  JL_LOCK(&codegen_lock);
  fptr.fptr = li->fptr;
  fptr.jlcall_api = li->jlcall_api;
- if (fptr.fptr) {
+ if (fptr.fptr && fptr.jlcall_api) {
  JL_UNLOCK(&codegen_lock);
  return fptr;
  }
  assert(F);
  assert(!li->inCompile);
  fptr.fptr = (jl_fptr_t)getAddressForFunction(F);
- fptr.jlcall_api = (F->getFunctionType() == jl_func_sig ? 0 : 1);
+ fptr.jlcall_api = jl_jlcall_api(F);
  assert(fptr.fptr != NULL);
  // decide if the fptr should be cached somewhere also
  if (li->functionObjectsDecls.functionObject == F) {
  if (li->fptr) {
  // don't change fptr as that leads to race conditions
- // with the update to jlcall_api
+ // with the (not) simultaneous update to jlcall_api
  }
- else if (li->inferred || fptr.jlcall_api != 0) {
+ else if (li->inferred || fptr.jlcall_api != 1) {
  li->jlcall_api = fptr.jlcall_api;
  li->fptr = fptr.fptr;
  }
@@ -1048,7 +1060,7 @@ jl_generic_fptr_t jl_generate_fptr(jl_lambda_info_t *li, void *_F)
  jl_lambda_info_t *unspec = li->def->unspecialized;
  if (unspec->fptr) {
  // don't change fptr as that leads to race conditions
- // with the update to jlcall_api
+ // with the (not) simultaneous update to jlcall_api
  }
  else if (unspec->functionObjectsDecls.functionObject == F) {
  unspec->jlcall_api = fptr.jlcall_api;
@@ -2799,8 +2811,8 @@ static jl_cgval_t emit_invoke(jl_expr_t *ex, jl_codectx_t *ctx)
  return mark_julia_const(li->inferred);
  }
  if (decls.functionObject) {
- int jlcall_api = ((Function*)decls.functionObject)->getFunctionType() == jl_func_sig ? 0 : 1;
- if (jlcall_api == 0) {
+ int jlcall_api = jl_jlcall_api(decls.functionObject);
+ if (jlcall_api == 1) {
  jl_cgval_t fval = emit_expr(args[1], ctx);
  jl_cgval_t result = emit_call_function_object(li, fval, decls, &args[1], nargs - 1, (jl_value_t*)ex, ctx);
  if (result.typ == jl_bottom_type)
@@ -3450,7 +3462,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t
  if (lam) {
  name = jl_symbol_name(lam->def->name);
  jl_llvm_functions_t decls = jl_compile_linfo(lam, NULL);
- if (decls.functionObject == NULL || lam->jlcall_api != 0) {
+ if (decls.functionObject == NULL || lam->jlcall_api != 1) {
  lam = NULL; // TODO: use emit_invoke framework to dispatch these
  }
  else {
@@ -5055,7 +5067,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs
  add_named_global(f, fptr);
  }
  else {
- if (lam->jlcall_api != 0) { // jl_func_sig_sparams -- don't bother emitting the FunctionObject (since can't be used right now)
+ if (lam->jlcall_api != 1) { // jl_func_sig_sparams -- don't bother emitting the FunctionObject (since can't be used right now)
  assert(lam->fptr == NULL);
  lam->fptr = fptr;
  }

src/codegen_internal.h

@@ -1,7 +1,5 @@
 // This file is a part of Julia. License is MIT: http:https://julialang.org/license
 
-extern FunctionType *jl_func_sig;
-
 #if defined(LLVM38) && !defined(LLVM37)
 # include <llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h>
 void notifyObjectLoaded(RTDyldMemoryManager *memmgr,

src/debuginfo.cpp

@@ -258,8 +258,8 @@ class JuliaJITEventListener: public JITEventListener
  linfo_in_flight.erase(linfo_it);
  if (!linfo->fptr && linfo->functionObjectsDecls.functionObject &&
  ((Function*)linfo->functionObjectsDecls.functionObject)->getName().equals(sName)) {
- int jlcall_api = (F.getFunctionType() == jl_func_sig ? 0 : 1);
- if (linfo->inferred || jlcall_api != 0) {
+ int jlcall_api = jl_jlcall_api(&F);
+ if (linfo->inferred || jlcall_api != 1) {
  linfo->jlcall_api = jlcall_api;
  linfo->fptr = (jl_fptr_t)(uintptr_t)Code;
  }
@@ -494,8 +494,8 @@ class JuliaJITEventListener: public JITEventListener
  linfo_in_flight.erase(linfo_it);
  Function *F = (Function*)linfo->functionObjectsDecls.functionObject;
  if (!linfo->fptr && F && F->getName().equals(sName)) {
- int jlcall_api = (F->getFunctionType() == jl_func_sig ? 0 : 1);
- if (linfo->inferred || jlcall_api != 0) {
+ int jlcall_api = jl_jlcall_api(F);
+ if (linfo->inferred || jlcall_api != 1) {
  linfo->jlcall_api = jlcall_api;
  linfo->fptr = (jl_fptr_t)(uintptr_t)Addr;
  }
@@ -577,8 +577,8 @@ class JuliaJITEventListener: public JITEventListener
  linfo_in_flight.erase(linfo_it);
  Function *F = (Function*)linfo->functionObjectsDecls.functionObject;
  if (!linfo->fptr && F && F->getName().equals(sName)) {
- int jlcall_api = (F->getFunctionType() == jl_func_sig ? 0 : 1);
- if (linfo->inferred || jlcall_api != 0) {
+ int jlcall_api = jl_jlcall_api(F);
+ if (linfo->inferred || jlcall_api != 1) {
  linfo->jlcall_api = jlcall_api;
  linfo->fptr = (jl_fptr_t)(uintptr_t)Addr;
  }

src/dump.c

@@ -452,13 +452,13 @@ static void jl_update_all_fptrs(void)
 
 // --- serialize ---
 
-static void jl_serialize_fptr(jl_serializer_state *s, void *fptr)
+static uint16_t jl_fptr_id(void *fptr)
 {
  void **pbp = ptrhash_bp(&fptr_to_id, fptr);
  if (*pbp == HT_NOTFOUND || fptr == NULL)
- write_uint16(s->s, 1);
+ return 1;
  else
- write_uint16(s->s, *(intptr_t*)pbp);
+ return *(intptr_t*)pbp;
 }
 
 static int module_in_worklist(jl_module_t *mod)
@@ -925,13 +925,25 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v)
  jl_serialize_value(s, li->rettype);
  jl_serialize_value(s, (jl_value_t*)li->sparam_vals);
  jl_serialize_value(s, (jl_value_t*)li->def);
- jl_serialize_fptr(s, (void*)(uintptr_t)li->fptr);
- // save functionObject pointers
- write_int8(s->s, li->jlcall_api);
- if (li->jlcall_api != 2) {
+ uint16_t id = jl_fptr_id((void*)(uintptr_t)li->fptr);
+ if (li->jlcall_api == 2) {
+ write_int8(s->s, 2);
+ }
+ else if (id >= 2) {
+ write_int8(s->s, -li->jlcall_api);
+ write_uint16(s->s, id);
+ }
+ else if (li->functionObjectsDecls.functionObject) {
+ int jlcall_api = jl_jlcall_api(li->functionObjectsDecls.functionObject);
+ assert(jlcall_api);
+ // save functionObject pointers
+ write_int8(s->s, jlcall_api);
  write_int32(s->s, jl_assign_functionID(li->functionObjectsDecls.functionObject));
  write_int32(s->s, jl_assign_functionID(li->functionObjectsDecls.specFunctionObject));
  }
+ else {
+ write_int8(s->s, 0);
+ }
  }
  else if (jl_typeis(v, jl_module_type)) {
  jl_serialize_module(s, (jl_module_t*)v);
@@ -1588,18 +1600,26 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta
  li->def = (jl_method_t*)jl_deserialize_value(s, (jl_value_t**)&li->def);
  if (li->def)
  jl_gc_wb(li, li->def);
- li->fptr = NULL;
  li->functionObjectsDecls.functionObject = NULL;
  li->functionObjectsDecls.specFunctionObject = NULL;
  li->inInference = 0;
  li->inCompile = 0;
- li->fptr = jl_deserialize_fptr(s);
- li->jlcall_api = read_int8(s->s);
- if (li->jlcall_api != 2) {
+ int8_t jlcall_api = read_int8(s->s);
+ if (jlcall_api == 2 || jlcall_api == 0) {
+ li->fptr = NULL;
+ li->jlcall_api = jlcall_api;
+ }
+ else if (jlcall_api < 0) {
+ li->fptr = jl_deserialize_fptr(s);
+ li->jlcall_api = -jlcall_api;
+ }
+ else {
  int32_t cfunc_llvm, func_llvm;
  func_llvm = read_int32(s->s);
  cfunc_llvm = read_int32(s->s);
  jl_delayed_fptrs(li, func_llvm, cfunc_llvm);
+ li->fptr = NULL;
+ li->jlcall_api = jlcall_api;
  }
  li->compile_traced = 0;
  return (jl_value_t*)li;

src/gf.c

@@ -164,6 +164,7 @@ void jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_t fptr)
  }
  jl_lambda_info_t *li = jl_new_lambda_info_uninit();
  li->fptr = fptr;
+ li->jlcall_api = 1;
  li->specTypes = jl_anytuple_type;
 
  li->def = jl_new_method_uninit();
@@ -229,7 +230,8 @@ JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettyp
  jl_gc_wb(li, rettype);
  li->inferred = inferred;
  jl_gc_wb(li, inferred);
- li->jlcall_api = (const_api == jl_true ? 2 : 0);
+ if (const_api == jl_true)
+ li->jlcall_api = 2;
 }
 
 static int jl_is_uninferred(jl_lambda_info_t *li)
@@ -1248,7 +1250,6 @@ static jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method)
 
 JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int include_ambiguous);
 
-// TODO: setting jlcall_api needs store ordering WRT fptr and inferred
 jl_llvm_functions_t jl_compile_for_dispatch(jl_lambda_info_t *li)
 {
  if (li->jlcall_api == 2)

src/julia.h

@@ -193,9 +193,10 @@ typedef jl_value_t *(*jl_fptr_linfo_t)(struct _jl_lambda_info_t*, jl_value_t**,
 typedef struct {
  union {
  jl_fptr_t fptr;
- jl_fptr_t fptr0;
- jl_fptr_sparam_t fptr1;
- jl_fptr_linfo_t fptr3;
+ jl_fptr_t fptr1;
+ // constant fptr2;
+ jl_fptr_sparam_t fptr3;
+ jl_fptr_linfo_t fptr4;
  };
  uint8_t jlcall_api;
 } jl_generic_fptr_t;
@@ -278,7 +279,7 @@ typedef struct _jl_lambda_info_t {
  uint8_t jlcall_api; // the c-abi for fptr; 0 = jl_fptr_t, 1 = jl_fptr_sparam_t, 2 = constval
  uint8_t compile_traced; // if set will notify callback if this linfo is compiled
  jl_fptr_t fptr; // jlcall entry point with api specified by jlcall_api
- jl_fptr_t unspecialized_ducttape; // if template can't be compiled due to intrinsics, an un-inferred fptr may get stored here, jlcall_api = 0
+ jl_fptr_t unspecialized_ducttape; // if template can't be compiled due to intrinsics, an un-inferred fptr may get stored here, jlcall_api = 1
 
  // On the old JIT, handles to all Functions generated for this linfo
  // For the new JITs, handles to declarations in the shadow module

src/julia_internal.h

@@ -187,7 +187,7 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val
  fptr.jlcall_api = meth->jlcall_api;
  if (fptr.jlcall_api == 2)
  return meth->inferred;
- if (__unlikely(fptr.fptr == NULL)) {
+ if (__unlikely(fptr.fptr == NULL || fptr.jlcall_api == 0)) {
  // first see if it likely needs to be compiled
  void *F = meth->functionObjectsDecls.functionObject;
  if (!F) // ask codegen to try to turn it into llvm code
@@ -197,7 +197,7 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val
  // if it hasn't been inferred, try using the unspecialized meth cache instead
  if (!meth->inferred) {
  fptr.fptr = meth->unspecialized_ducttape;
- fptr.jlcall_api = 0;
+ fptr.jlcall_api = 1;
  if (!fptr.fptr) {
  if (meth->def && !meth->def->isstaged && meth->def->unspecialized) {
  fptr.fptr = meth->def->unspecialized->fptr;
@@ -207,19 +207,19 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val
  }
  }
  }
- if (!fptr.fptr) {
+ if (!fptr.fptr || fptr.jlcall_api == 0) {
  // ask codegen to make the fptr
  fptr = jl_generate_fptr(meth, F);
  if (fptr.jlcall_api == 2)
  return meth->inferred;
  }
  }
- if (fptr.jlcall_api == 0)
- return fptr.fptr0(args[0], &args[1], nargs-1);
- else if (fptr.jlcall_api == 1)
- return fptr.fptr1(meth->sparam_vals, args[0], &args[1], nargs-1);
+ if (fptr.jlcall_api == 1)
+ return fptr.fptr1(args[0], &args[1], nargs-1);
  else if (fptr.jlcall_api == 3)
- return fptr.fptr3(meth, &args[0], nargs, meth->sparam_vals);
+ return fptr.fptr3(meth->sparam_vals, args[0], &args[1], nargs-1);
+ else if (fptr.jlcall_api == 4)
+ return fptr.fptr4(meth, &args[0], nargs, meth->sparam_vals);
  else
  abort();
 }
@@ -450,6 +450,7 @@ static inline void jl_set_gc_and_wait(void)
 void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len);
 int32_t jl_get_llvm_gv(jl_value_t *p);
 int32_t jl_assign_functionID(/*llvm::Function*/void *function);
+int32_t jl_jlcall_api(/*llvm::Function*/const void *function);
 // the first argument to jl_idtable_rehash is used to return a value
 // make sure it is rooted if it is used after the function returns
 JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz);