diff --git a/src/alloc.c b/src/alloc.c index 30940dc41a0c4..4006c97061929 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -456,15 +456,15 @@ STATIC_INLINE jl_value_t *jl_call_staged(jl_svec_t *sparam_vals, jl_lambda_info_ jl_generic_fptr_t fptr; fptr.fptr = generator->fptr; fptr.jlcall_api = generator->jlcall_api; - if (__unlikely(fptr.fptr == NULL)) { + if (__unlikely(fptr.fptr == NULL || fptr.jlcall_api == 0)) { void *F = jl_compile_linfo(generator, (jl_source_info_t*)generator->inferred).functionObject; fptr = jl_generate_fptr(generator, F); } assert(jl_svec_len(generator->def->sparam_syms) == jl_svec_len(sparam_vals)); - if (fptr.jlcall_api == 0) - return fptr.fptr0(args[0], &args[1], nargs-1); - else if (fptr.jlcall_api == 1) - return fptr.fptr1(sparam_vals, args[0], &args[1], nargs-1); + if (fptr.jlcall_api == 1) + return fptr.fptr1(args[0], &args[1], nargs-1); + else if (fptr.jlcall_api == 3) + return fptr.fptr3(sparam_vals, args[0], &args[1], nargs-1); else abort(); // shouldn't have inferred any other calling convention } diff --git a/src/anticodegen.c b/src/anticodegen.c index 948c2a81b08c3..1ba10ceb6e3b9 100644 --- a/src/anticodegen.c +++ b/src/anticodegen.c @@ -47,5 +47,5 @@ jl_value_t *jl_interpret_call(jl_lambda_info_t *lam, jl_value_t **args, uint32_t void jl_generate_fptr(jl_lambda_info_t *li) { li->fptr = (jl_fptr_t)&jl_interpret_call; - li->jlcall_api = 3; + li->jlcall_api = 4; } diff --git a/src/codegen.cpp b/src/codegen.cpp index d131659aa16ce..f879c2bc9efb4 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -203,7 +203,7 @@ static Type *T_jlvalue; static Type *T_pjlvalue; static Type *T_ppjlvalue; static Type *jl_parray_llvmt; -FunctionType *jl_func_sig; +static FunctionType *jl_func_sig; static FunctionType *jl_func_sig_sparams; static Type *T_pvoidfunc; @@ -279,6 +279,18 @@ DICompositeType jl_di_func_null_sig; #endif #endif + +extern "C" +int32_t jl_jlcall_api(const void *function) +{ + // give the function an index in the constant lookup table + if (function == NULL) + return 0; + Function *F = (Function*)function; + return (F->getFunctionType() == jl_func_sig ? 1 : 3); +} + + // constants static Value *V_null; static Type *NoopType; @@ -1010,33 +1022,33 @@ jl_generic_fptr_t jl_generate_fptr(jl_lambda_info_t *li, void *_F) jl_generic_fptr_t fptr; fptr.fptr = li->fptr; fptr.jlcall_api = li->jlcall_api; - if (fptr.fptr) { + if (fptr.fptr && fptr.jlcall_api) { return fptr; } fptr.fptr = li->unspecialized_ducttape; - fptr.jlcall_api = 0; + fptr.jlcall_api = 1; if (!li->inferred && fptr.fptr) { return fptr; } JL_LOCK(&codegen_lock); fptr.fptr = li->fptr; fptr.jlcall_api = li->jlcall_api; - if (fptr.fptr) { + if (fptr.fptr && fptr.jlcall_api) { JL_UNLOCK(&codegen_lock); return fptr; } assert(F); assert(!li->inCompile); fptr.fptr = (jl_fptr_t)getAddressForFunction(F); - fptr.jlcall_api = (F->getFunctionType() == jl_func_sig ? 0 : 1); + fptr.jlcall_api = jl_jlcall_api(F); assert(fptr.fptr != NULL); // decide if the fptr should be cached somewhere also if (li->functionObjectsDecls.functionObject == F) { if (li->fptr) { // don't change fptr as that leads to race conditions - // with the update to jlcall_api + // with the (not) simultaneous update to jlcall_api } - else if (li->inferred || fptr.jlcall_api != 0) { + else if (li->inferred || fptr.jlcall_api != 1) { li->jlcall_api = fptr.jlcall_api; li->fptr = fptr.fptr; } @@ -1048,7 +1060,7 @@ jl_generic_fptr_t jl_generate_fptr(jl_lambda_info_t *li, void *_F) jl_lambda_info_t *unspec = li->def->unspecialized; if (unspec->fptr) { // don't change fptr as that leads to race conditions - // with the update to jlcall_api + // with the (not) simultaneous update to jlcall_api } else if (unspec->functionObjectsDecls.functionObject == F) { unspec->jlcall_api = fptr.jlcall_api; @@ -2799,8 +2811,8 @@ static jl_cgval_t emit_invoke(jl_expr_t *ex, jl_codectx_t *ctx) return mark_julia_const(li->inferred); } if (decls.functionObject) { - int jlcall_api = ((Function*)decls.functionObject)->getFunctionType() == jl_func_sig ? 0 : 1; - if (jlcall_api == 0) { + int jlcall_api = jl_jlcall_api(decls.functionObject); + if (jlcall_api == 1) { jl_cgval_t fval = emit_expr(args[1], ctx); jl_cgval_t result = emit_call_function_object(li, fval, decls, &args[1], nargs - 1, (jl_value_t*)ex, ctx); if (result.typ == jl_bottom_type) @@ -3450,7 +3462,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t if (lam) { name = jl_symbol_name(lam->def->name); jl_llvm_functions_t decls = jl_compile_linfo(lam, NULL); - if (decls.functionObject == NULL || lam->jlcall_api != 0) { + if (decls.functionObject == NULL || lam->jlcall_api != 1) { lam = NULL; // TODO: use emit_invoke framework to dispatch these } else { @@ -5055,7 +5067,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs add_named_global(f, fptr); } else { - if (lam->jlcall_api != 0) { // jl_func_sig_sparams -- don't bother emitting the FunctionObject (since can't be used right now) + if (lam->jlcall_api != 1) { // jl_func_sig_sparams -- don't bother emitting the FunctionObject (since can't be used right now) assert(lam->fptr == NULL); lam->fptr = fptr; } diff --git a/src/codegen_internal.h b/src/codegen_internal.h index fdc56b23b1cf2..12ab6fb2bf645 100644 --- a/src/codegen_internal.h +++ b/src/codegen_internal.h @@ -1,7 +1,5 @@ // This file is a part of Julia. License is MIT: http://julialang.org/license -extern FunctionType *jl_func_sig; - #if defined(LLVM38) && !defined(LLVM37) # include void notifyObjectLoaded(RTDyldMemoryManager *memmgr, diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index a372e38d6a8a5..418159b789afb 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -258,8 +258,8 @@ class JuliaJITEventListener: public JITEventListener linfo_in_flight.erase(linfo_it); if (!linfo->fptr && linfo->functionObjectsDecls.functionObject && ((Function*)linfo->functionObjectsDecls.functionObject)->getName().equals(sName)) { - int jlcall_api = (F.getFunctionType() == jl_func_sig ? 0 : 1); - if (linfo->inferred || jlcall_api != 0) { + int jlcall_api = jl_jlcall_api(&F); + if (linfo->inferred || jlcall_api != 1) { linfo->jlcall_api = jlcall_api; linfo->fptr = (jl_fptr_t)(uintptr_t)Code; } @@ -494,8 +494,8 @@ class JuliaJITEventListener: public JITEventListener linfo_in_flight.erase(linfo_it); Function *F = (Function*)linfo->functionObjectsDecls.functionObject; if (!linfo->fptr && F && F->getName().equals(sName)) { - int jlcall_api = (F->getFunctionType() == jl_func_sig ? 0 : 1); - if (linfo->inferred || jlcall_api != 0) { + int jlcall_api = jl_jlcall_api(F); + if (linfo->inferred || jlcall_api != 1) { linfo->jlcall_api = jlcall_api; linfo->fptr = (jl_fptr_t)(uintptr_t)Addr; } @@ -577,8 +577,8 @@ class JuliaJITEventListener: public JITEventListener linfo_in_flight.erase(linfo_it); Function *F = (Function*)linfo->functionObjectsDecls.functionObject; if (!linfo->fptr && F && F->getName().equals(sName)) { - int jlcall_api = (F->getFunctionType() == jl_func_sig ? 0 : 1); - if (linfo->inferred || jlcall_api != 0) { + int jlcall_api = jl_jlcall_api(F); + if (linfo->inferred || jlcall_api != 1) { linfo->jlcall_api = jlcall_api; linfo->fptr = (jl_fptr_t)(uintptr_t)Addr; } diff --git a/src/dump.c b/src/dump.c index a6a359749d5a8..d2fdc955b5c1a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -452,13 +452,13 @@ static void jl_update_all_fptrs(void) // --- serialize --- -static void jl_serialize_fptr(jl_serializer_state *s, void *fptr) +static uint16_t jl_fptr_id(void *fptr) { void **pbp = ptrhash_bp(&fptr_to_id, fptr); if (*pbp == HT_NOTFOUND || fptr == NULL) - write_uint16(s->s, 1); + return 1; else - write_uint16(s->s, *(intptr_t*)pbp); + return *(intptr_t*)pbp; } static int module_in_worklist(jl_module_t *mod) @@ -925,13 +925,25 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v) jl_serialize_value(s, li->rettype); jl_serialize_value(s, (jl_value_t*)li->sparam_vals); jl_serialize_value(s, (jl_value_t*)li->def); - jl_serialize_fptr(s, (void*)(uintptr_t)li->fptr); - // save functionObject pointers - write_int8(s->s, li->jlcall_api); - if (li->jlcall_api != 2) { + uint16_t id = jl_fptr_id((void*)(uintptr_t)li->fptr); + if (li->jlcall_api == 2) { + write_int8(s->s, 2); + } + else if (id >= 2) { + write_int8(s->s, -li->jlcall_api); + write_uint16(s->s, id); + } + else if (li->functionObjectsDecls.functionObject) { + int jlcall_api = jl_jlcall_api(li->functionObjectsDecls.functionObject); + assert(jlcall_api); + // save functionObject pointers + write_int8(s->s, jlcall_api); write_int32(s->s, jl_assign_functionID(li->functionObjectsDecls.functionObject)); write_int32(s->s, jl_assign_functionID(li->functionObjectsDecls.specFunctionObject)); } + else { + write_int8(s->s, 0); + } } else if (jl_typeis(v, jl_module_type)) { jl_serialize_module(s, (jl_module_t*)v); @@ -1588,18 +1600,26 @@ static jl_value_t *jl_deserialize_value_(jl_serializer_state *s, jl_value_t *vta li->def = (jl_method_t*)jl_deserialize_value(s, (jl_value_t**)&li->def); if (li->def) jl_gc_wb(li, li->def); - li->fptr = NULL; li->functionObjectsDecls.functionObject = NULL; li->functionObjectsDecls.specFunctionObject = NULL; li->inInference = 0; li->inCompile = 0; - li->fptr = jl_deserialize_fptr(s); - li->jlcall_api = read_int8(s->s); - if (li->jlcall_api != 2) { + int8_t jlcall_api = read_int8(s->s); + if (jlcall_api == 2 || jlcall_api == 0) { + li->fptr = NULL; + li->jlcall_api = jlcall_api; + } + else if (jlcall_api < 0) { + li->fptr = jl_deserialize_fptr(s); + li->jlcall_api = -jlcall_api; + } + else { int32_t cfunc_llvm, func_llvm; func_llvm = read_int32(s->s); cfunc_llvm = read_int32(s->s); jl_delayed_fptrs(li, func_llvm, cfunc_llvm); + li->fptr = NULL; + li->jlcall_api = jlcall_api; } li->compile_traced = 0; return (jl_value_t*)li; diff --git a/src/gf.c b/src/gf.c index a065333b8f5c8..7ea98ac54ab8a 100644 --- a/src/gf.c +++ b/src/gf.c @@ -164,6 +164,7 @@ void jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_t fptr) } jl_lambda_info_t *li = jl_new_lambda_info_uninit(); li->fptr = fptr; + li->jlcall_api = 1; li->specTypes = jl_anytuple_type; li->def = jl_new_method_uninit(); @@ -229,7 +230,8 @@ JL_DLLEXPORT void jl_set_lambda_rettype(jl_lambda_info_t *li, jl_value_t *rettyp jl_gc_wb(li, rettype); li->inferred = inferred; jl_gc_wb(li, inferred); - li->jlcall_api = (const_api == jl_true ? 2 : 0); + if (const_api == jl_true) + li->jlcall_api = 2; } static int jl_is_uninferred(jl_lambda_info_t *li) @@ -1248,7 +1250,6 @@ static jl_lambda_info_t *jl_get_unspecialized(jl_lambda_info_t *method) JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_tupletype_t *types, int lim, int include_ambiguous); -// TODO: setting jlcall_api needs store ordering WRT fptr and inferred jl_llvm_functions_t jl_compile_for_dispatch(jl_lambda_info_t *li) { if (li->jlcall_api == 2) diff --git a/src/julia.h b/src/julia.h index e6caa8c87cb9f..5086ea5ad9aeb 100644 --- a/src/julia.h +++ b/src/julia.h @@ -193,9 +193,10 @@ typedef jl_value_t *(*jl_fptr_linfo_t)(struct _jl_lambda_info_t*, jl_value_t**, typedef struct { union { jl_fptr_t fptr; - jl_fptr_t fptr0; - jl_fptr_sparam_t fptr1; - jl_fptr_linfo_t fptr3; + jl_fptr_t fptr1; + // constant fptr2; + jl_fptr_sparam_t fptr3; + jl_fptr_linfo_t fptr4; }; uint8_t jlcall_api; } jl_generic_fptr_t; @@ -278,7 +279,7 @@ typedef struct _jl_lambda_info_t { uint8_t jlcall_api; // the c-abi for fptr; 0 = jl_fptr_t, 1 = jl_fptr_sparam_t, 2 = constval uint8_t compile_traced; // if set will notify callback if this linfo is compiled jl_fptr_t fptr; // jlcall entry point with api specified by jlcall_api - jl_fptr_t unspecialized_ducttape; // if template can't be compiled due to intrinsics, an un-inferred fptr may get stored here, jlcall_api = 0 + jl_fptr_t unspecialized_ducttape; // if template can't be compiled due to intrinsics, an un-inferred fptr may get stored here, jlcall_api = 1 // On the old JIT, handles to all Functions generated for this linfo // For the new JITs, handles to declarations in the shadow module diff --git a/src/julia_internal.h b/src/julia_internal.h index 137c90b0451d9..ba33e92ee7cd2 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -187,7 +187,7 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val fptr.jlcall_api = meth->jlcall_api; if (fptr.jlcall_api == 2) return meth->inferred; - if (__unlikely(fptr.fptr == NULL)) { + if (__unlikely(fptr.fptr == NULL || fptr.jlcall_api == 0)) { // first see if it likely needs to be compiled void *F = meth->functionObjectsDecls.functionObject; if (!F) // ask codegen to try to turn it into llvm code @@ -197,7 +197,7 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val // if it hasn't been inferred, try using the unspecialized meth cache instead if (!meth->inferred) { fptr.fptr = meth->unspecialized_ducttape; - fptr.jlcall_api = 0; + fptr.jlcall_api = 1; if (!fptr.fptr) { if (meth->def && !meth->def->isstaged && meth->def->unspecialized) { fptr.fptr = meth->def->unspecialized->fptr; @@ -207,19 +207,19 @@ STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_val } } } - if (!fptr.fptr) { + if (!fptr.fptr || fptr.jlcall_api == 0) { // ask codegen to make the fptr fptr = jl_generate_fptr(meth, F); if (fptr.jlcall_api == 2) return meth->inferred; } } - if (fptr.jlcall_api == 0) - return fptr.fptr0(args[0], &args[1], nargs-1); - else if (fptr.jlcall_api == 1) - return fptr.fptr1(meth->sparam_vals, args[0], &args[1], nargs-1); + if (fptr.jlcall_api == 1) + return fptr.fptr1(args[0], &args[1], nargs-1); else if (fptr.jlcall_api == 3) - return fptr.fptr3(meth, &args[0], nargs, meth->sparam_vals); + return fptr.fptr3(meth->sparam_vals, args[0], &args[1], nargs-1); + else if (fptr.jlcall_api == 4) + return fptr.fptr4(meth, &args[0], nargs, meth->sparam_vals); else abort(); } @@ -450,6 +450,7 @@ static inline void jl_set_gc_and_wait(void) void jl_dump_native(const char *bc_fname, const char *obj_fname, const char *sysimg_data, size_t sysimg_len); int32_t jl_get_llvm_gv(jl_value_t *p); int32_t jl_assign_functionID(/*llvm::Function*/void *function); +int32_t jl_jlcall_api(/*llvm::Function*/const void *function); // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz);