diff --git a/src/ccall.cpp b/src/ccall.cpp index 6b28aa152ac9f..d3c7e13f03ec4 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -366,6 +366,7 @@ static Value *llvm_type_rewrite( // --- argument passing and scratch space utilities --- +// Returns T_prjlvalue static Value *runtime_apply_type_env(jl_codectx_t &ctx, jl_value_t *ty) { // box if concrete type was not statically known @@ -441,7 +442,7 @@ static Value *julia_to_native( // We're passing Any if (toboxed) { assert(!byRef); // don't expect any ABI to pass pointers by pointer - return maybe_decay_untracked(boxed(ctx, jvinfo)); + return boxed(ctx, jvinfo); } assert(jl_is_datatype(jlto) && julia_struct_has_layout((jl_datatype_t*)jlto, jlto_env)); @@ -1028,6 +1029,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar // --- code generator for ccall itself --- +// Returns T_prjlvalue static Value *box_ccall_result(jl_codectx_t &ctx, Value *result, Value *runtime_dt, jl_value_t *rt) { // XXX: need to handle parameterized zero-byte types (singleton) @@ -1278,7 +1280,7 @@ static bool verify_ref_type(jl_codectx_t &ctx, jl_value_t* ref, jl_unionall_t *u else { Value *notany = ctx.builder.CreateICmpNE( boxed(ctx, runtime_sp), - maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jl_any_type))); + track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_any_type))); error_unless(ctx, notany, make_errmsg(fname, n, rt_err_msg_notany)); always_error = false; } @@ -1729,7 +1731,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) emit_typeof_boxed(ctx, val), val.isghost ? ConstantPointerNull::get(T_pint8_derived) : ctx.builder.CreateBitCast( - decay_derived(data_pointer(ctx, val)), + decay_derived(ctx, data_pointer(ctx, val)), T_pint8_derived) }; Value *ret = ctx.builder.CreateCall(prepare_call(jl_object_id__func), makeArrayRef(args)); @@ -1795,7 +1797,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( v = julia_to_native(ctx, largty, toboxed, jargty_in_env, unionall_env, arg, byRef, ai); bool issigned = jl_signed_type && jl_subtype(jargty, (jl_value_t*)jl_signed_type); if (byRef) { - v = decay_derived(v); + v = decay_derived(ctx, v); // julia_to_native should already have done the alloca and store assert(v->getType() == pargty); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 231f97c0082cb..bf6775b5d01c4 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -10,61 +10,49 @@ static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) return load_or_store; } -// Take an arbitrary untracked value and make it gc-tracked -static Value *maybe_decay_untracked(IRBuilder<> &irbuilder, Value *V) +static Value *track_pjlvalue(jl_codectx_t &ctx, Value *V) { - if (V->getType() == T_pjlvalue) - return irbuilder.CreateAddrSpaceCast(V, T_prjlvalue); - else if (V->getType() == T_ppjlvalue) - return irbuilder.CreateBitCast(V, T_pprjlvalue); - return V; + assert(V->getType() == T_pjlvalue); + return ctx.builder.CreateAddrSpaceCast(V, T_prjlvalue); } -// Take an untracked value and make it tracked (specialized on Constant) -static Constant *maybe_decay_untracked(IRBuilder<> &irbuilder, Constant *C) +// Take an arbitrary untracked value and make it gc-tracked +static Value *maybe_decay_untracked(jl_codectx_t &ctx, Value *V) { - if (C->getType() == T_pjlvalue) - return ConstantExpr::getAddrSpaceCast(C, T_prjlvalue); - else if (C->getType() == T_ppjlvalue) - return ConstantExpr::getBitCast(C, T_pprjlvalue); - return C; + if (V->getType() == T_pjlvalue) + return ctx.builder.CreateAddrSpaceCast(V, T_prjlvalue); + assert(V->getType() == T_prjlvalue); + return V; } // Take any value and mark that it may be derived from a rooted value -static Value *decay_derived(IRBuilder<> &irbuilder, Value *V) +static Value *decay_derived(jl_codectx_t &ctx, Value *V) { Type *T = V->getType(); if (cast(T)->getAddressSpace() == AddressSpace::Derived) return V; // Once llvm deletes pointer element types, we won't need it here any more either. Type *NewT = PointerType::get(cast(T)->getElementType(), AddressSpace::Derived); - return irbuilder.CreateAddrSpaceCast(V, NewT); + return ctx.builder.CreateAddrSpaceCast(V, NewT); } // Take any value and make it safe to pass to GEP -static Value *maybe_decay_tracked(IRBuilder<> &irbuilder, Value *V) +static Value *maybe_decay_tracked(jl_codectx_t &ctx, Value *V) { Type *T = V->getType(); if (cast(T)->getAddressSpace() != AddressSpace::Tracked) return V; Type *NewT = PointerType::get(cast(T)->getElementType(), AddressSpace::Derived); - return irbuilder.CreateAddrSpaceCast(V, NewT); + return ctx.builder.CreateAddrSpaceCast(V, NewT); } -static Value *mark_callee_rooted(IRBuilder<> &irbuilder, Value *V) +static Value *mark_callee_rooted(jl_codectx_t &ctx, Value *V) { assert(V->getType() == T_pjlvalue || V->getType() == T_prjlvalue); - return irbuilder.CreateAddrSpaceCast(V, + return ctx.builder.CreateAddrSpaceCast(V, PointerType::get(T_jlvalue, AddressSpace::CalleeRooted)); } -#define maybe_decay_untracked(V) maybe_decay_untracked(ctx.builder, (V)) -#define maybe_decay_untracked(V) maybe_decay_untracked(ctx.builder, (V)) -#define decay_derived(V) decay_derived(ctx.builder, (V)) -#define maybe_decay_tracked(V) maybe_decay_tracked(ctx.builder, (V)) -#define mark_callee_rooted(V) mark_callee_rooted(ctx.builder, (V)) - - // --- language feature checks --- #define JL_FEAT_TEST(ctx, feature) ((ctx).params->feature) @@ -193,7 +181,7 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) unsigned AS = cast(V->getType())->getAddressSpace(); if (AS != AddressSpace::Tracked && AS != AddressSpace::Derived) return V; - V = decay_derived(V); + V = decay_derived(ctx, V); Type *T = PointerType::get(T_jlvalue, AddressSpace::Derived); if (V->getType() != T) V = ctx.builder.CreateBitCast(V, T); @@ -391,6 +379,7 @@ static inline Instruction *maybe_mark_load_dereferenceable(Instruction *LI, bool return maybe_mark_load_dereferenceable(LI, can_be_null, size, alignment); } +// Returns T_pjlvalue static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p) { if (p == NULL) @@ -403,6 +392,7 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p) false, jl_typeof(p))); } +// Returns T_pjlvalue static Value *literal_pointer_val(jl_codectx_t &ctx, jl_binding_t *p) { // emit a pointer to any jl_value_t which will be valid across reloading code @@ -796,7 +786,7 @@ static Value *emit_nthptr_addr(jl_codectx_t &ctx, Value *v, ssize_t n, bool gctr { return ctx.builder.CreateInBoundsGEP( T_prjlvalue, - emit_bitcast(ctx, maybe_decay_tracked(v), T_pprjlvalue), + emit_bitcast(ctx, maybe_decay_tracked(ctx, v), T_pprjlvalue), ConstantInt::get(T_size, n)); } @@ -804,7 +794,7 @@ static Value *emit_nthptr_addr(jl_codectx_t &ctx, Value *v, Value *idx) { return ctx.builder.CreateInBoundsGEP( T_prjlvalue, - emit_bitcast(ctx, maybe_decay_tracked(v), T_pprjlvalue), + emit_bitcast(ctx, maybe_decay_tracked(ctx, v), T_pprjlvalue), idx); } @@ -824,6 +814,7 @@ static Value *emit_nthptr_recast(jl_codectx_t &ctx, Value *v, ssize_t n, MDNode static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &v); +// Returns T_prjlvalue static Value *emit_typeof(jl_codectx_t &ctx, Value *tt) { assert(tt != NULL && !isa(tt) && "expected a conditionally boxed value"); @@ -863,7 +854,7 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) ptr = literal_pointer_val_slot(ctx, (jl_value_t*)jt); } else { - ptr = maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jt)); + ptr = track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jt)); } datatype_or_p = ctx.builder.CreateSelect(cmp, ptr, datatype_or_p); }, @@ -871,8 +862,8 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) counter); auto emit_unboxty = [&] () -> Value* { if (imaging_mode) - return maybe_decay_untracked( - tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_pjlvalue, datatype_or_p, sizeof(void*)))); + return track_pjlvalue( + ctx, tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_pjlvalue, datatype_or_p, sizeof(void*)))); return datatype_or_p; }; Value *res; @@ -904,6 +895,7 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) return mark_julia_const(p.typ); } +// Returns T_prjlvalue static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p) { return boxed(ctx, emit_typeof(ctx, p)); @@ -911,7 +903,7 @@ static Value *emit_typeof_boxed(jl_codectx_t &ctx, const jl_cgval_t &p) static Value *emit_datatype_types(jl_codectx_t &ctx, Value *dt) { - Value *Ptr = emit_bitcast(ctx, decay_derived(dt), T_ppjlvalue); + Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), T_ppjlvalue); Value *Idx = ConstantInt::get(T_size, offsetof(jl_datatype_t, types) / sizeof(void*)); return tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad( T_pjlvalue, ctx.builder.CreateInBoundsGEP(T_pjlvalue, Ptr, Idx), sizeof(void*))); @@ -925,7 +917,7 @@ static Value *emit_datatype_nfields(jl_codectx_t &ctx, Value *dt) static Value *emit_datatype_size(jl_codectx_t &ctx, Value *dt) { - Value *Ptr = emit_bitcast(ctx, decay_derived(dt), T_pint32); + Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), T_pint32); Value *Idx = ConstantInt::get(T_size, offsetof(jl_datatype_t, size) / sizeof(int)); return tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_int32, ctx.builder.CreateInBoundsGEP(T_int32, Ptr, Idx), sizeof(int32_t))); } @@ -982,7 +974,7 @@ static Value *emit_sizeof(jl_codectx_t &ctx, const jl_cgval_t &p) static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) { - Value *Ptr = emit_bitcast(ctx, decay_derived(dt), T_pint8); + Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), T_pint8); Value *Idx = ConstantInt::get(T_size, offsetof(jl_datatype_t, mutabl)); Value *mutabl = tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_int8, ctx.builder.CreateInBoundsGEP(T_int8, Ptr, Idx), 1)); @@ -992,7 +984,7 @@ static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) /* this is valid code, it's simply unused static Value *emit_datatype_abstract(jl_codectx_t &ctx, Value *dt) { - Value *Ptr = emit_bitcast(ctx, decay_derived(dt), T_pint8); + Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), T_pint8); Value *Idx = ConstantInt::get(T_size, offsetof(jl_datatype_t, abstract)); Value *abstract = tbaa_decorate(tbaa_const, @@ -1054,7 +1046,7 @@ static void raise_exception(jl_codectx_t &ctx, Value *exc, jl_box_voidpointer(wrap(ctx.builder.GetInsertBlock())), jl_box_voidpointer(wrap(exc))); } else { - ctx.builder.CreateCall(prepare_call(jlthrow_func), { mark_callee_rooted(exc) }); + ctx.builder.CreateCall(prepare_call(jlthrow_func), { mark_callee_rooted(ctx, exc) }); } ctx.builder.CreateUnreachable(); if (!contBB) { @@ -1087,7 +1079,7 @@ static void emit_type_error(jl_codectx_t &ctx, const jl_cgval_t &x, Value *type, { Value *msg_val = stringConstPtr(ctx.emission_context, ctx.builder, msg); ctx.builder.CreateCall(prepare_call(jltypeerror_func), - { msg_val, maybe_decay_untracked(type), mark_callee_rooted(boxed(ctx, x))}); + { msg_val, maybe_decay_untracked(ctx, type), mark_callee_rooted(ctx, boxed(ctx, x))}); } static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, jl_value_t *type, const std::string *msg) @@ -1120,8 +1112,8 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, // intersection with Type needs to be handled specially if (jl_has_intersect_type_not_kind(type) || jl_has_intersect_type_not_kind(intersected_type)) { - Value *vx = maybe_decay_untracked(boxed(ctx, x)); - Value *vtyp = maybe_decay_untracked(literal_pointer_val(ctx, type)); + Value *vx = boxed(ctx, x); + Value *vtyp = track_pjlvalue(ctx, literal_pointer_val(ctx, type)); if (msg && *msg == "typeassert") { ctx.builder.CreateCall(prepare_call(jltypeassert_func), { vx, vtyp }); return std::make_pair(ConstantInt::get(T_int1, 1), true); @@ -1148,7 +1140,7 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, ctx.builder.CreateCondBr(isboxed, isaBB, postBB); ctx.builder.SetInsertPoint(isaBB); Value *istype_boxed = ctx.builder.CreateICmpEQ(emit_typeof(ctx, x.Vboxed), - maybe_decay_untracked(literal_pointer_val(ctx, intersected_type))); + track_pjlvalue(ctx, literal_pointer_val(ctx, intersected_type))); ctx.builder.CreateBr(postBB); isaBB = ctx.builder.GetInsertBlock(); // could have changed ctx.builder.SetInsertPoint(postBB); @@ -1162,7 +1154,7 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, } } return std::make_pair(ctx.builder.CreateICmpEQ(emit_typeof_boxed(ctx, x), - maybe_decay_untracked(literal_pointer_val(ctx, intersected_type))), false); + track_pjlvalue(ctx, literal_pointer_val(ctx, intersected_type))), false); } jl_datatype_t *dt = (jl_datatype_t*)jl_unwrap_unionall(intersected_type); if (jl_is_datatype(dt) && !dt->abstract && jl_subtype(dt->name->wrapper, type)) { @@ -1170,15 +1162,15 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, // so the isa test reduces to a comparison of the typename by pointer return std::make_pair( ctx.builder.CreateICmpEQ( - mark_callee_rooted(emit_datatype_name(ctx, emit_typeof_boxed(ctx, x))), - mark_callee_rooted(literal_pointer_val(ctx, (jl_value_t*)dt->name))), + mark_callee_rooted(ctx, emit_datatype_name(ctx, emit_typeof_boxed(ctx, x))), + mark_callee_rooted(ctx, literal_pointer_val(ctx, (jl_value_t*)dt->name))), false); } // everything else can be handled via subtype tests return std::make_pair(ctx.builder.CreateICmpNE( ctx.builder.CreateCall(prepare_call(jlsubtype_func), - { maybe_decay_untracked(emit_typeof_boxed(ctx, x)), - maybe_decay_untracked(literal_pointer_val(ctx, type)) }), + { emit_typeof_boxed(ctx, x), + track_pjlvalue(ctx, literal_pointer_val(ctx, type)) }), ConstantInt::get(T_int32, 0)), false); } @@ -1204,7 +1196,7 @@ static void emit_typecheck(jl_codectx_t &ctx, const jl_cgval_t &x, jl_value_t *t static Value *emit_isconcrete(jl_codectx_t &ctx, Value *typ) { Value *isconcrete; - isconcrete = ctx.builder.CreateConstInBoundsGEP1_32(T_int8, emit_bitcast(ctx, decay_derived(typ), T_pint8), offsetof(jl_datatype_t, isconcretetype)); + isconcrete = ctx.builder.CreateConstInBoundsGEP1_32(T_int8, emit_bitcast(ctx, decay_derived(ctx, typ), T_pint8), offsetof(jl_datatype_t, isconcretetype)); isconcrete = tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_int8, isconcrete, 1)); isconcrete = ctx.builder.CreateTrunc(isconcrete, T_int1); return isconcrete; @@ -1246,7 +1238,7 @@ static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_v ctx.builder.CreateCall(prepare_call(jlvboundserror_func), { ainfo.V, len, i }); } else if (ainfo.isboxed) { // jl_datatype_t or boxed jl_value_t - ctx.builder.CreateCall(prepare_call(jlboundserror_func), { mark_callee_rooted(boxed(ctx, ainfo)), i }); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), { mark_callee_rooted(ctx, boxed(ctx, ainfo)), i }); } else { // unboxed jl_value_t* Value *a = ainfo.V; @@ -1260,7 +1252,7 @@ static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_v a = tempSpace; } ctx.builder.CreateCall(prepare_call(jluboundserror_func), { - emit_bitcast(ctx, decay_derived(a), T_pint8), + emit_bitcast(ctx, decay_derived(ctx, a), T_pint8), literal_pointer_val(ctx, ty), i }); } @@ -1378,7 +1370,7 @@ static void typed_store(jl_codectx_t &ctx, if (!isboxed) r = emit_unbox(ctx, elty, rhs, jltype); else - r = maybe_decay_untracked(boxed(ctx, rhs)); + r = boxed(ctx, rhs); Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); if (ptr->getType() != ptrty) ptr = ctx.builder.CreateBitCast(ptr, ptrty); @@ -1405,6 +1397,7 @@ static void typed_store(jl_codectx_t &ctx, // --- convert boolean value to julia --- +// Returns T_pjlvalue static Value *julia_bool(jl_codectx_t &ctx, Value *cond) { return ctx.builder.CreateSelect(cond, literal_pointer_val(ctx, jl_true), @@ -1607,7 +1600,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, } Value *fldptr = ctx.builder.CreateInBoundsGEP( T_prjlvalue, - maybe_decay_tracked(emit_bitcast(ctx, data_pointer(ctx, strct), T_pprjlvalue)), + maybe_decay_tracked(ctx, emit_bitcast(ctx, data_pointer(ctx, strct), T_pprjlvalue)), idx0()); LoadInst *fld = ctx.builder.CreateAlignedLoad(T_prjlvalue, fldptr, sizeof(void*)); fld->setOrdering(AtomicOrdering::Unordered); @@ -1622,7 +1615,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, assert(nfields > 0); // nf == 0 trapped by all_pointers case jl_value_t *jft = jl_svecref(stt->types, 0); idx = idx0(); - Value *ptr = maybe_decay_tracked(data_pointer(ctx, strct)); + Value *ptr = maybe_decay_tracked(ctx, data_pointer(ctx, strct)); if (!stt->mutabl && !(maybe_null && jft == (jl_value_t*)jl_bool_type)) { // just compute the pointer and let user load it when necessary Type *fty = julia_type_to_llvm(ctx, jft); @@ -1655,7 +1648,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st bool maybe_null = idx >= (unsigned)jt->ninitialized; size_t byte_offset = jl_field_offset(jt, idx); if (strct.ispointer()) { - Value *staddr = maybe_decay_tracked(data_pointer(ctx, strct)); + Value *staddr = maybe_decay_tracked(ctx, data_pointer(ctx, strct)); bool isboxed; Type *lt = julia_type_to_llvm(ctx, (jl_value_t*)jt, &isboxed); Value *addr; @@ -1843,7 +1836,7 @@ static Value *emit_arraylen_prim(jl_codectx_t &ctx, const jl_cgval_t &tinfo) jl_value_t *ty = tinfo.typ; #ifdef STORE_ARRAY_LEN Value *addr = ctx.builder.CreateStructGEP(jl_array_llvmt, - emit_bitcast(ctx, decay_derived(t), jl_parray_llvmt), + emit_bitcast(ctx, decay_derived(ctx, t), jl_parray_llvmt), 1); //index (not offset) of length field in jl_parray_llvmt MDNode *tbaa = arraytype_constshape(ty) ? tbaa_const : tbaa_arraylen; LoadInst *len = ctx.builder.CreateAlignedLoad(addr, sizeof(size_t)); @@ -1903,13 +1896,13 @@ static Value *emit_arrayptr_internal(jl_codectx_t &ctx, const jl_cgval_t &tinfo, static Value *emit_arrayptr(jl_codectx_t &ctx, const jl_cgval_t &tinfo, bool isboxed = false) { Value *t = boxed(ctx, tinfo); - return emit_arrayptr_internal(ctx, tinfo, decay_derived(t), AddressSpace::Loaded, isboxed); + return emit_arrayptr_internal(ctx, tinfo, decay_derived(ctx, t), AddressSpace::Loaded, isboxed); } static Value *emit_unsafe_arrayptr(jl_codectx_t &ctx, const jl_cgval_t &tinfo, bool isboxed = false) { Value *t = boxed(ctx, tinfo); - t = emit_pointer_from_objref(ctx, decay_derived(t)); + t = emit_pointer_from_objref(ctx, decay_derived(ctx, t)); return emit_arrayptr_internal(ctx, tinfo, t, 0, isboxed); } @@ -1933,7 +1926,7 @@ static Value *emit_arrayflags(jl_codectx_t &ctx, const jl_cgval_t &tinfo) #endif Value *addr = ctx.builder.CreateStructGEP( jl_array_llvmt, - emit_bitcast(ctx, decay_derived(t), jl_parray_llvmt), + emit_bitcast(ctx, decay_derived(ctx, t), jl_parray_llvmt), arrayflag_field); return tbaa_decorate(tbaa_arrayflags, ctx.builder.CreateAlignedLoad(T_int16, addr, sizeof(int16_t))); } @@ -1956,7 +1949,7 @@ static Value *emit_arrayelsize(jl_codectx_t &ctx, const jl_cgval_t &tinfo) int elsize_field = 2; #endif Value *addr = ctx.builder.CreateStructGEP(jl_array_llvmt, - emit_bitcast(ctx, decay_derived(t), jl_parray_llvmt), + emit_bitcast(ctx, decay_derived(ctx, t), jl_parray_llvmt), elsize_field); return tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_int16, addr, sizeof(int16_t))); } @@ -1974,7 +1967,7 @@ static Value *emit_arrayoffset(jl_codectx_t &ctx, const jl_cgval_t &tinfo, int n Value *addr = ctx.builder.CreateStructGEP( jl_array_llvmt, - emit_bitcast(ctx, decay_derived(t), jl_parray_llvmt), + emit_bitcast(ctx, decay_derived(ctx, t), jl_parray_llvmt), offset_field); return tbaa_decorate(tbaa_arrayoffset, ctx.builder.CreateAlignedLoad(T_int32, addr, sizeof(int32_t))); } @@ -2070,7 +2063,7 @@ static Value *emit_array_nd_index( ctx.builder.CreateAlignedStore(idxs[k], ctx.builder.CreateInBoundsGEP(T_size, tmp, ConstantInt::get(T_size, k)), sizeof(size_t)); } ctx.builder.CreateCall(prepare_call(jlboundserrorv_func), - { mark_callee_rooted(a), tmp, ConstantInt::get(T_size, nidxs) }); + { mark_callee_rooted(ctx, a), tmp, ConstantInt::get(T_size, nidxs) }); ctx.builder.CreateUnreachable(); ctx.f->getBasicBlockList().push_back(endBB); @@ -2194,20 +2187,21 @@ static Value *as_value(jl_codectx_t &ctx, Type *to, const jl_cgval_t &v) } // some types have special boxing functions with small-value caches +// Returns T_prjlvalue static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t) { jl_value_t *jt = vinfo.typ; if (jt == (jl_value_t*)jl_bool_type) - return julia_bool(ctx, ctx.builder.CreateTrunc(as_value(ctx, t, vinfo), T_int1)); + return track_pjlvalue(ctx, julia_bool(ctx, ctx.builder.CreateTrunc(as_value(ctx, t, vinfo), T_int1))); if (t == T_int1) - return julia_bool(ctx, as_value(ctx, t, vinfo)); + return track_pjlvalue(ctx, julia_bool(ctx, as_value(ctx, t, vinfo))); if (ctx.linfo && jl_is_method(ctx.linfo->def.method) && !vinfo.ispointer()) { // don't bother codegen pre-boxing for toplevel if (Constant *c = dyn_cast(vinfo.V)) { jl_value_t *s = static_constant_instance(c, jt); if (s) { jl_add_method_root(ctx, s); - return literal_pointer_val(ctx, s); + return track_pjlvalue(ctx, literal_pointer_val(ctx, s)); } } } @@ -2216,7 +2210,7 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t assert(jl_is_datatype(jb)); Value *box = NULL; if (jb == jl_int8_type) - box = call_with_attrs(ctx, box_int8_func, as_value(ctx, t, vinfo)); + box = track_pjlvalue(ctx, call_with_attrs(ctx, box_int8_func, as_value(ctx, t, vinfo))); else if (jb == jl_int16_type) box = call_with_attrs(ctx, box_int16_func, as_value(ctx, t, vinfo)); else if (jb == jl_int32_type) @@ -2229,7 +2223,7 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t // box = ctx.builder.CreateCall(box_float64_func, as_value(ctx, t, vinfo); // for Float64, fall through to generic case below, to inline alloc & init of Float64 box. cheap, I know. else if (jb == jl_uint8_type) - box = call_with_attrs(ctx, box_uint8_func, as_value(ctx, t, vinfo)); + box = track_pjlvalue(ctx, call_with_attrs(ctx, box_uint8_func, as_value(ctx, t, vinfo))); else if (jb == jl_uint16_type) box = call_with_attrs(ctx, box_uint16_func, as_value(ctx, t, vinfo)); else if (jb == jl_uint32_type) @@ -2248,7 +2242,7 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t else if (!jb->abstract && jl_datatype_nbits(jb) == 0) { // singleton assert(jb->instance != NULL); - return literal_pointer_val(ctx, jb->instance); + return track_pjlvalue(ctx, literal_pointer_val(ctx, jb->instance)); } return box; } @@ -2260,7 +2254,7 @@ static Value *compute_box_tindex(jl_codectx_t &ctx, Value *datatype, jl_value_t for_each_uniontype_small( [&](unsigned idx, jl_datatype_t *jt) { if (jl_subtype((jl_value_t*)jt, supertype)) { - Value *cmp = ctx.builder.CreateICmpEQ(maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jt)), datatype); + Value *cmp = ctx.builder.CreateICmpEQ(track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jt)), datatype); tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, idx), tindex); } }, @@ -2332,6 +2326,7 @@ static AllocaInst *try_emit_union_alloca(jl_codectx_t &ctx, jl_uniontype_t *ut, * case, the calling code must separately deal with the case where * `vinfo` is already an unknown boxed union (union tag 0x80). */ +// Returns T_prjlvalue static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallBitVector &skip) { // given vinfo::Union{T, S}, emit IR of the form: @@ -2366,7 +2361,7 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB switchInst->addCase(ConstantInt::get(T_int8, idx), tempBB); Value *box; if (type_is_ghost(t)) { - box = literal_pointer_val(ctx, jt->instance); + box = track_pjlvalue(ctx, literal_pointer_val(ctx, jt->instance)); } else { jl_cgval_t vinfo_r = jl_cgval_t(vinfo, (jl_value_t*)jt, NULL); @@ -2376,7 +2371,6 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB init_bits_cgval(ctx, box, vinfo_r, jl_is_mutable(jt) ? tbaa_mutab : tbaa_immut); } } - box = maybe_decay_untracked(box); tempBB = ctx.builder.GetInsertBlock(); // could have changed box_merge->addIncoming(box, tempBB); ctx.builder.CreateBr(postBB); @@ -2386,8 +2380,7 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB ctx.builder.SetInsertPoint(defaultBB); if (skip.size() > 0) { assert(skip[0]); - Value *box = maybe_decay_untracked(V_null); - box_merge->addIncoming(box, defaultBB); + box_merge->addIncoming(V_rnull, defaultBB); ctx.builder.CreateBr(postBB); } else if (!vinfo.Vboxed) { @@ -2408,6 +2401,7 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB // this is used to wrap values for generic contexts, where a // dynamically-typed value is required (e.g. argument to unknown function). // if it's already a pointer it's left alone. +// Returns T_prjlvalue static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo) { jl_value_t *jt = vinfo.typ; @@ -2415,12 +2409,13 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo) // We have an undef value on a (hopefully) dead branch return UndefValue::get(T_prjlvalue); if (vinfo.constant) - return maybe_decay_untracked(literal_pointer_val(ctx, vinfo.constant)); + return track_pjlvalue(ctx, literal_pointer_val(ctx, vinfo.constant)); // This can happen in early bootstrap for `gc_preserve_begin` return value. if (jt == (jl_value_t*)jl_nothing_type) - return maybe_decay_untracked(literal_pointer_val(ctx, jl_nothing)); + return track_pjlvalue(ctx, literal_pointer_val(ctx, jl_nothing)); if (vinfo.isboxed) { assert(vinfo.V == vinfo.Vboxed); + assert(vinfo.V->getType() == T_prjlvalue); return vinfo.V; } @@ -2439,9 +2434,6 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo) box = emit_allocobj(ctx, jl_datatype_size(jt), literal_pointer_val(ctx, (jl_value_t*)jt)); init_bits_cgval(ctx, box, vinfo, jl_is_mutable(jt) ? tbaa_mutab : tbaa_immut); } - else { - box = maybe_decay_untracked(box); - } } return box; } @@ -2542,8 +2534,8 @@ static void emit_cpointercheck(jl_codectx_t &ctx, const jl_cgval_t &x, const std emit_typecheck(ctx, mark_julia_type(ctx, t, true, jl_any_type), (jl_value_t*)jl_datatype_type, msg); Value *istype = - ctx.builder.CreateICmpEQ(mark_callee_rooted(emit_datatype_name(ctx, t)), - mark_callee_rooted(literal_pointer_val(ctx, (jl_value_t*)jl_pointer_typename))); + ctx.builder.CreateICmpEQ(mark_callee_rooted(ctx, emit_datatype_name(ctx, t)), + mark_callee_rooted(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_pointer_typename))); BasicBlock *failBB = BasicBlock::Create(jl_LLVMContext,"fail",ctx.f); BasicBlock *passBB = BasicBlock::Create(jl_LLVMContext,"pass"); ctx.builder.CreateCondBr(istype, passBB, failBB); @@ -2561,7 +2553,7 @@ static Value *emit_allocobj(jl_codectx_t &ctx, size_t static_size, Value *jt) { Value *ptls_ptr = emit_bitcast(ctx, ctx.ptlsStates, T_pint8); Function *F = prepare_call(jl_alloc_obj_func); - auto call = ctx.builder.CreateCall(F, {ptls_ptr, ConstantInt::get(T_size, static_size), maybe_decay_untracked(jt)}); + auto call = ctx.builder.CreateCall(F, {ptls_ptr, ConstantInt::get(T_size, static_size), maybe_decay_untracked(ctx, jt)}); call->setAttributes(F->getAttributes()); return call; } @@ -2585,9 +2577,9 @@ static void emit_write_barrier(jl_codectx_t &ctx, Value *parent, Value *ptr) static void emit_write_barrier(jl_codectx_t &ctx, Value *parent, ArrayRef ptrs) { SmallVector decay_ptrs; - decay_ptrs.push_back(maybe_decay_untracked(emit_bitcast(ctx, parent, T_prjlvalue))); + decay_ptrs.push_back(maybe_decay_untracked(ctx, emit_bitcast(ctx, parent, T_prjlvalue))); for (auto ptr : ptrs) { - decay_ptrs.push_back(maybe_decay_untracked(emit_bitcast(ctx, ptr, T_prjlvalue))); + decay_ptrs.push_back(maybe_decay_untracked(ctx, emit_bitcast(ctx, ptr, T_prjlvalue))); } ctx.builder.CreateCall(prepare_call(jl_write_barrier_func), decay_ptrs); } @@ -2610,12 +2602,12 @@ static void emit_setfield(jl_codectx_t &ctx, if (byte_offset > 0) { addr = ctx.builder.CreateInBoundsGEP( T_int8, - emit_bitcast(ctx, maybe_decay_tracked(addr), T_pint8), + emit_bitcast(ctx, maybe_decay_tracked(ctx, addr), T_pint8), ConstantInt::get(T_size, byte_offset)); // TODO: use emit_struct_gep } jl_value_t *jfty = jl_svecref(sty->types, idx0); if (jl_field_isptr(sty, idx0)) { - Value *r = maybe_decay_untracked(boxed(ctx, rhs)); // don't need a temporary gcroot since it'll be rooted by strct + Value *r = boxed(ctx, rhs); // don't need a temporary gcroot since it'll be rooted by strct cast(tbaa_decorate(strct.tbaa, ctx.builder.CreateAlignedStore(r, emit_bitcast(ctx, addr, T_pprjlvalue), sizeof(jl_value_t*)))) @@ -2631,7 +2623,7 @@ static void emit_setfield(jl_codectx_t &ctx, return; Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jfty); tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1)); - Value *ptindex = ctx.builder.CreateInBoundsGEP(T_int8, emit_bitcast(ctx, maybe_decay_tracked(addr), T_pint8), ConstantInt::get(T_size, fsz)); + Value *ptindex = ctx.builder.CreateInBoundsGEP(T_int8, emit_bitcast(ctx, maybe_decay_tracked(ctx, addr), T_pint8), ConstantInt::get(T_size, fsz)); tbaa_decorate(tbaa_unionselbyte, ctx.builder.CreateAlignedStore(tindex, ptindex, 1)); // copy data if (!rhs.isghost) { @@ -2808,7 +2800,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg Value *strct = emit_allocobj(ctx, jl_datatype_size(sty), literal_pointer_val(ctx, (jl_value_t*)ty)); jl_cgval_t strctinfo = mark_julia_type(ctx, strct, true, ty); - strct = decay_derived(strct); + strct = decay_derived(ctx, strct); undef_derived_strct(ctx.builder, strct, sty, strctinfo.tbaa); for (size_t i = nargs; i < nf; i++) { if (!jl_field_isptr(sty, i) && jl_is_uniontype(jl_field_type(sty, i))) { diff --git a/src/codegen.cpp b/src/codegen.cpp index dbce9f8c116a2..15b2631e133f3 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -234,6 +234,7 @@ static DISubroutineType *jl_di_func_null_sig; // constants static Constant *V_null; +static Constant *V_rnull; static bool type_is_ghost(Type *ty) { return (ty == T_void || ty->isEmptyTy()); @@ -907,6 +908,7 @@ struct jl_cgval_t { // runtime values) if `(TIndex | 0x80) != 0`, then `Vboxed == V` (by value). // For convenience, we also set this value of isboxed values, in which case // it is equal (at compile time) to V. + // If this is non-NULL, it is always of type `T_prjlvalue` Value *Vboxed; Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8 jl_value_t *constant; // constant value (rooted in linfo.def.roots) @@ -929,6 +931,8 @@ struct jl_cgval_t { isghost(false), tbaa(isboxed ? best_tbaa(typ) : nullptr) { + if (Vboxed) + assert(Vboxed->getType() == T_prjlvalue); assert(gcroot == nullptr); assert(!(isboxed && TIndex != NULL)); assert(TIndex == NULL || TIndex->getType() == T_int8); @@ -957,6 +961,8 @@ struct jl_cgval_t { isghost(v.isghost), tbaa(v.tbaa) { + if (Vboxed) + assert(Vboxed->getType() == T_prjlvalue); // this constructor expects we had a badly or equivalently typed version // make sure we aren't discarding the actual type information if (v.TIndex) { @@ -1161,10 +1167,9 @@ static void undef_derived_strct(IRBuilder<> &irbuilder, Value *ptr, jl_datatype_ if (np == 0) return; ptr = irbuilder.CreateBitCast(ptr, T_prjlvalue->getPointerTo(ptr->getType()->getPointerAddressSpace())); - Value *V_null = ConstantPointerNull::get(cast(T_prjlvalue)); for (i = 0; i < np; i++) { Value *fld = irbuilder.CreateConstInBoundsGEP1_32(T_prjlvalue, ptr, jl_ptr_offset(sty, i)); - tbaa_decorate(tbaa, irbuilder.CreateStore(V_null, fld)); + tbaa_decorate(tbaa, irbuilder.CreateStore(V_rnull, fld)); } } @@ -1461,7 +1466,7 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & if (old_idx == 0) { // didn't handle this item before, select its new union index maybe_setup_union_isa(); - Value *cmp = ctx.builder.CreateICmpEQ(maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jt)), union_box_dt); + Value *cmp = ctx.builder.CreateICmpEQ(track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jt)), union_box_dt); union_box_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, 0x80 | idx), union_box_tindex); } }, @@ -1511,9 +1516,10 @@ static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t & tbaa = tbaa_stack; } slotv = ctx.builder.CreateSelect(isboxv, - decay_derived(boxv), - decay_derived(emit_bitcast(ctx, slotv, boxv->getType()))); + decay_derived(ctx, boxv), + decay_derived(ctx, emit_bitcast(ctx, slotv, boxv->getType()))); jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex); + assert(boxv->getType() == T_prjlvalue); newv.Vboxed = boxv; newv.tbaa = tbaa; return newv; @@ -2277,7 +2283,7 @@ static jl_cgval_t emit_getfield(jl_codectx_t &ctx, const jl_cgval_t &strct, jl_s strct, mark_julia_const((jl_value_t*)name) }; - Value *result = emit_jlcall(ctx, jlgetfield_func, maybe_decay_untracked(V_null), myargs_array, 2, JLCALL_F_CC); + Value *result = emit_jlcall(ctx, jlgetfield_func, V_rnull, myargs_array, 2, JLCALL_F_CC); return mark_julia_type(ctx, result, true, jl_any_type); } @@ -2288,18 +2294,18 @@ static Value *emit_box_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const Value *varg2 = arg2.constant ? literal_pointer_val(ctx, arg2.constant) : arg2.V; assert(varg1 && varg2 && (arg1.isboxed || arg1.TIndex) && (arg2.isboxed || arg2.TIndex) && "Only boxed types are valid for pointer comparison."); - varg1 = maybe_decay_tracked(varg1); - varg2 = maybe_decay_tracked(varg2); + varg1 = maybe_decay_tracked(ctx, varg1); + varg2 = maybe_decay_tracked(ctx, varg2); if (cast(varg1->getType())->getAddressSpace() != cast(varg2->getType())->getAddressSpace()) { - varg1 = decay_derived(varg1); - varg2 = decay_derived(varg2); + varg1 = decay_derived(ctx, varg1); + varg2 = decay_derived(ctx, varg2); } return ctx.builder.CreateICmpEQ(emit_bitcast(ctx, varg1, T_pint8), emit_bitcast(ctx, varg2, T_pint8)); } - Value *varg1 = mark_callee_rooted(boxed(ctx, arg1)); - Value *varg2 = mark_callee_rooted(boxed(ctx, arg2)); + Value *varg1 = mark_callee_rooted(ctx, boxed(ctx, arg1)); + Value *varg2 = mark_callee_rooted(ctx, boxed(ctx, arg2)); return ctx.builder.CreateTrunc(ctx.builder.CreateCall(prepare_call(jlegal_func), {varg1, varg2}), T_int1); } @@ -2376,8 +2382,8 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a if (at->isAggregateType()) { // Struct or Array jl_datatype_t *sty = (jl_datatype_t*)arg1.typ; size_t sz = jl_datatype_size(sty); - Value *varg1 = arg1.ispointer() ? maybe_decay_tracked(data_pointer(ctx, arg1)) : arg1.V; - Value *varg2 = arg2.ispointer() ? maybe_decay_tracked(data_pointer(ctx, arg2)) : arg2.V; + Value *varg1 = arg1.ispointer() ? maybe_decay_tracked(ctx, data_pointer(ctx, arg1)) : arg1.V; + Value *varg2 = arg2.ispointer() ? maybe_decay_tracked(ctx, data_pointer(ctx, arg2)) : arg2.V; if (sz > 512 && !sty->layout->haspadding) { if (!arg1.ispointer()) varg1 = value_to_pointer(ctx, arg1).V; @@ -2495,8 +2501,8 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva // which is already enough to ensure pointer uniqueness for this test // even if the other pointer managed to get garbage collected return ctx.builder.CreateICmpEQ( - mark_callee_rooted(boxed(ctx, arg1)), - mark_callee_rooted(boxed(ctx, arg2))); + mark_callee_rooted(ctx, boxed(ctx, arg1)), + mark_callee_rooted(ctx, boxed(ctx, arg2))); } if (jl_type_intersection(rt1, rt2) == (jl_value_t*)jl_bottom_type) // types are disjoint (exhaustive test) @@ -2607,7 +2613,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, // turn Core._apply(f, Tuple) ==> f(Tuple...) using the jlcall calling convention if Tuple is the va allocation if (LoadInst *load = dyn_cast_or_null(argv[arg_start].V)) { if (load->getPointerOperand() == ctx.slots[ctx.vaSlot].boxroot && ctx.argArray) { - Value *theF = maybe_decay_untracked(boxed(ctx, argv[arg_start-1])); + Value *theF = boxed(ctx, argv[arg_start-1]); Value *nva = emit_n_varargs(ctx); #ifdef _P64 nva = ctx.builder.CreateTrunc(nva, T_int32); @@ -2782,7 +2788,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else { PHINode *data_owner = NULL; // owner object against which the write barrier must check if (isboxed || (jl_is_datatype(ety) && ((jl_datatype_t*)ety)->layout->npointers > 0)) { // if elements are just bits, don't need a write barrier - Value *aryv = maybe_decay_untracked(boxed(ctx, ary)); + Value *aryv = boxed(ctx, ary); Value *flags = emit_arrayflags(ctx, ary); // the owner of the data is ary itself except if ary->how == 3 flags = ctx.builder.CreateAnd(flags, 3); @@ -2797,7 +2803,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (jl_is_long(ndp)) { own_ptr = ctx.builder.CreateAlignedLoad(T_prjlvalue, ctx.builder.CreateConstGEP1_32(T_prjlvalue, - emit_bitcast(ctx, decay_derived(aryv), T_pprjlvalue), + emit_bitcast(ctx, decay_derived(ctx, aryv), T_pprjlvalue), jl_array_data_owner_offset(nd) / sizeof(jl_value_t*)), sizeof(void*)); tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type)); @@ -2928,7 +2934,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, jl_true); } bool isboxed = !jl_datatype_isinlinealloc(jt); - Value *ptr = maybe_decay_tracked(data_pointer(ctx, obj)); + Value *ptr = maybe_decay_tracked(ctx, data_pointer(ctx, obj)); *ret = typed_load(ctx, ptr, vidx, isboxed ? (jl_value_t*)jl_any_type : jt, obj.tbaa, nullptr, false); @@ -3012,7 +3018,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, Value *idx = emit_unbox(ctx, T_size, fld, (jl_value_t*)jl_long_type); jl_value_t *boundscheck = (nargs == 3 ? argv[3].constant : jl_true); emit_bounds_check(ctx, typ, (jl_value_t*)jl_datatype_type, idx, types_len, boundscheck); - Value *fieldtyp_p = ctx.builder.CreateInBoundsGEP(T_prjlvalue, decay_derived(emit_bitcast(ctx, types_svec, T_pprjlvalue)), idx); + Value *fieldtyp_p = ctx.builder.CreateInBoundsGEP(T_prjlvalue, decay_derived(ctx, emit_bitcast(ctx, types_svec, T_pprjlvalue)), idx); Value *fieldtyp = tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_prjlvalue, fieldtyp_p, sizeof(void*))); *ret = mark_julia_type(ctx, fieldtyp, true, (jl_value_t*)jl_type_type); return true; @@ -3117,7 +3123,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, if (obj.ispointer()) { if (!jl_field_isptr(stt, fieldidx)) offs += ((jl_datatype_t*)jl_field_type(stt, fieldidx))->layout->first_ptr; - Value *ptr = emit_bitcast(ctx, maybe_decay_tracked(data_pointer(ctx, obj)), T_pprjlvalue); + Value *ptr = emit_bitcast(ctx, maybe_decay_tracked(ctx, data_pointer(ctx, obj)), T_pprjlvalue); Value *addr = ctx.builder.CreateConstInBoundsGEP1_32(T_prjlvalue, ptr, offs); // emit this using the same type as emit_getfield_knownidx // so that LLVM may be able to load-load forward them and fold the result @@ -3142,6 +3148,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return false; } +// Returns T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) { @@ -3153,7 +3160,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, argsT.push_back(T_prjlvalue); } for (size_t i = 0; i < nargs; i++) { - Value *arg = maybe_decay_untracked(boxed(ctx, argv[i])); + Value *arg = boxed(ctx, argv[i]); theArgs.push_back(arg); argsT.push_back(T_prjlvalue); } @@ -3165,6 +3172,7 @@ static CallInst *emit_jlcall(jl_codectx_t &ctx, Function *theFptr, Value *theF, result->setCallingConv(cc); return result; } +// Returns T_prjlvalue static CallInst *emit_jlcall(jl_codectx_t &ctx, JuliaFunction *theFptr, Value *theF, jl_cgval_t *argv, size_t nargs, CallingConv::ID cc) { @@ -3224,14 +3232,14 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_ jl_cgval_t arg = argv[i]; if (isboxed) { assert(at == T_prjlvalue && et == T_prjlvalue); - argvals[idx] = maybe_decay_untracked(boxed(ctx, arg)); + argvals[idx] = boxed(ctx, arg); } else if (et->isAggregateType()) { if (!arg.ispointer()) arg = value_to_pointer(ctx, arg); // can lazy load on demand, no copy needed assert(at == PointerType::get(et, AddressSpace::Derived)); - argvals[idx] = decay_derived(maybe_bitcast(ctx, + argvals[idx] = decay_derived(ctx, maybe_bitcast(ctx, data_pointer(ctx, arg), at)); } else { @@ -3268,8 +3276,8 @@ static jl_cgval_t emit_call_specfun_other(jl_codectx_t &ctx, jl_method_instance_ ctx.builder.CreateICmpEQ( ctx.builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), ConstantInt::get(T_int8, 0)), - decay_derived(ctx.builder.CreateBitCast(argvals[0], T_pjlvalue)), - decay_derived(box) + decay_derived(ctx, ctx.builder.CreateBitCast(argvals[0], T_pjlvalue)), + decay_derived(ctx, box) ); retval = mark_julia_slot(derived, jlretty, @@ -3429,7 +3437,7 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) // special case for known builtin not handled by emit_builtin_call auto it = builtin_func_map.find(jl_get_builtin_fptr(f.constant)); if (it != builtin_func_map.end()) { - Value *ret = emit_jlcall(ctx, it->second, maybe_decay_untracked(V_null), &argv[1], nargs - 1, JLCALL_F_CC); + Value *ret = emit_jlcall(ctx, it->second, V_rnull, &argv[1], nargs - 1, JLCALL_F_CC); return mark_julia_type(ctx, ret, true, rt); } } @@ -3448,7 +3456,7 @@ static void undef_var_error_ifnot(jl_codectx_t &ctx, Value *ok, jl_sym_t *name) ctx.builder.CreateCondBr(ok, ifok, err); ctx.builder.SetInsertPoint(err); ctx.builder.CreateCall(prepare_call(jlundefvarerror_func), - mark_callee_rooted(literal_pointer_val(ctx, (jl_value_t*)name))); + mark_callee_rooted(ctx, literal_pointer_val(ctx, (jl_value_t*)name))); ctx.builder.CreateUnreachable(); ctx.f->getBasicBlockList().push_back(ifok); ctx.builder.SetInsertPoint(ifok); @@ -3534,7 +3542,7 @@ static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i) i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); Value *sp = tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_prjlvalue, bp, sizeof(void*))); Value *isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp), - maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); + track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); jl_unionall_t *sparam = (jl_unionall_t*)ctx.linfo->def.method->sig; for (size_t j = 0; j < i; j++) { sparam = (jl_unionall_t*)sparam->body; @@ -3577,7 +3585,7 @@ static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) } if (vi.boxroot != NULL) { Value *boxed = ctx.builder.CreateAlignedLoad(T_prjlvalue, vi.boxroot, sizeof(void*), vi.isVolatile); - Value *box_isnull = ctx.builder.CreateICmpNE(boxed, maybe_decay_untracked(V_null)); + Value *box_isnull = ctx.builder.CreateICmpNE(boxed, V_rnull); if (vi.pTIndex) { // value is either boxed in the stack slot, or unboxed in value // as indicated by testing (pTIndex & 0x80) @@ -3608,7 +3616,7 @@ static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); Value *sp = tbaa_decorate(tbaa_const, ctx.builder.CreateAlignedLoad(T_prjlvalue, bp, sizeof(void*))); isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp), - maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); + track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); } else { jl_module_t *modu; @@ -3630,7 +3638,7 @@ static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) LoadInst *v = ctx.builder.CreateAlignedLoad(T_prjlvalue, bp, sizeof(void*)); tbaa_decorate(tbaa_binding, v); v->setOrdering(AtomicOrdering::Unordered); - isnull = ctx.builder.CreateICmpNE(v, maybe_decay_untracked(V_null)); + isnull = ctx.builder.CreateICmpNE(v, V_rnull); } else { Value *v = ctx.builder.CreateCall(prepare_call(jlboundp_func), { @@ -3685,7 +3693,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va Instruction *boxed = ctx.builder.CreateAlignedLoad(T_prjlvalue, vi.boxroot, sizeof(void*), vi.isVolatile); Value *box_isnull = NULL; if (vi.usedUndef) - box_isnull = ctx.builder.CreateICmpNE(boxed, maybe_decay_untracked(V_null)); + box_isnull = ctx.builder.CreateICmpNE(boxed, V_rnull); maybe_mark_load_dereferenceable(boxed, vi.usedUndef || vi.pTIndex, typ); if (vi.pTIndex) { // value is either boxed in the stack slot, or unboxed in value @@ -3697,7 +3705,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va isnull = ctx.builder.CreateSelect(load_unbox, isnull, box_isnull); if (v.V) { // v.V will be null if it is a union of all ghost values v.V = ctx.builder.CreateSelect(load_unbox, emit_bitcast(ctx, - decay_derived(v.V), boxed->getType()), decay_derived(boxed)); + decay_derived(ctx, v.V), boxed->getType()), decay_derived(ctx, boxed)); } else v.V = boxed; v.Vboxed = boxed; @@ -3825,8 +3833,8 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) #endif ctx.builder.CreateLifetimeEnd(dest); Value *ptr = ctx.builder.CreateSelect(isboxed, - maybe_bitcast(ctx, decay_derived(ptr_phi), T_pint8), - maybe_bitcast(ctx, decay_derived(phi), T_pint8)); + maybe_bitcast(ctx, decay_derived(ctx, ptr_phi), T_pint8), + maybe_bitcast(ctx, decay_derived(ctx, phi), T_pint8)); jl_cgval_t val = mark_julia_slot(ptr, phiType, Tindex_phi, tbaa_stack); // XXX: this TBAA is wrong for ptr_phi val.Vboxed = ptr_phi; ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, ptr_phi, r)); @@ -3953,14 +3961,15 @@ static void emit_varinfo_assign(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_cgval_t isboxed = ctx.builder.CreateICmpNE( ctx.builder.CreateAnd(rval_info.TIndex, ConstantInt::get(T_int8, 0x80)), ConstantInt::get(T_int8, 0)); - rval = maybe_decay_untracked(rval_info.Vboxed ? rval_info.Vboxed : V_null); + rval = rval_info.Vboxed ? rval_info.Vboxed : V_rnull; + assert(rval->getType() == T_prjlvalue); assert(!vi.value.constant); } else { assert(!vi.pTIndex || rval_info.isboxed || rval_info.constant); - rval = maybe_decay_untracked(boxed(ctx, rval_info)); + rval = boxed(ctx, rval_info); } - ctx.builder.CreateStore(maybe_decay_untracked(rval), vi.boxroot, vi.isVolatile); + ctx.builder.CreateStore(rval, vi.boxroot, vi.isVolatile); } // store unboxed variables @@ -3986,7 +3995,7 @@ static void emit_assignment(jl_codectx_t &ctx, jl_value_t *l, jl_value_t *r, ssi bp = global_binding_pointer(ctx, ctx.module, s, &bnd, true); if (bp != NULL) { // it's a global assert(bnd); - Value *rval = mark_callee_rooted(boxed(ctx, emit_expr(ctx, r, ssaval))); + Value *rval = mark_callee_rooted(ctx, boxed(ctx, emit_expr(ctx, r, ssaval))); ctx.builder.CreateCall(prepare_call(jlcheckassign_func), { literal_pointer_val(ctx, bnd), rval }); @@ -4023,7 +4032,7 @@ static Value *emit_condition(jl_codectx_t &ctx, const jl_cgval_t &condV, const s } if (condV.isboxed) { return ctx.builder.CreateICmpEQ(boxed(ctx, condV), - maybe_decay_untracked(literal_pointer_val(ctx, jl_false))); + track_pjlvalue(ctx, literal_pointer_val(ctx, jl_false))); } // not a boolean return ConstantInt::get(T_int1, 0); // TODO: replace with Undef @@ -4056,7 +4065,7 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result) // create a new uninitialized variable Value *lv = vi.boxroot; if (lv != NULL) - ctx.builder.CreateStore(maybe_decay_untracked(V_null), lv); + ctx.builder.CreateStore(V_rnull, lv); if (lv == NULL || vi.pTIndex != NULL) store_def_flag(ctx, vi, false); } @@ -4340,7 +4349,7 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval) } return mark_julia_type(ctx, ctx.builder.CreateCall(prepare_call(jlcopyast_func), - maybe_decay_untracked(boxed(ctx, ast))), true, jl_expr_type); + boxed(ctx, ast)), true, jl_expr_type); } else if (head == loopinfo_sym) { // parse Expr(:loopinfo, "julia.simdloop", ("llvm.loop.vectorize.width", 4)) @@ -4471,7 +4480,7 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod theFunc = prepare_call(jlinvoke_func); theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst->def); } - theFarg = maybe_decay_untracked(theFarg); + theFarg = track_pjlvalue(ctx, theFarg); auto args = f->arg_begin(); CallInst *r = ctx.builder.CreateCall(theFunc, { &*args, &*++args, &*++args, theFarg }); r->setAttributes(theFunc->getAttributes()); @@ -4762,7 +4771,7 @@ static Function* gen_cfun_wrapper( } else if (static_at || (!jl_is_typevar(jargty) && !jl_is_immutable_datatype(jargty))) { // must be a jl_value_t* (because it's mutable or contains gc roots) - inputarg = mark_julia_type(ctx, maybe_decay_untracked(emit_bitcast(ctx, val, T_prjlvalue)), true, jargty_proper); + inputarg = mark_julia_type(ctx, maybe_decay_untracked(ctx, emit_bitcast(ctx, val, T_prjlvalue)), true, jargty_proper); } else { // allocate val into a new box, if it might not be boxed @@ -4783,7 +4792,7 @@ static Function* gen_cfun_wrapper( ctx.builder.CreateCondBr(isrtboxed, boxedBB, loadBB); ctx.builder.SetInsertPoint(boxedBB); Value *p1 = ctx.builder.CreateBitCast(val, T_pjlvalue); - p1 = maybe_decay_untracked(p1); + p1 = track_pjlvalue(ctx, p1); ctx.builder.CreateBr(afterBB); ctx.builder.SetInsertPoint(loadBB); Value *isrtany = ctx.builder.CreateICmpEQ( @@ -4950,7 +4959,7 @@ static Function* gen_cfun_wrapper( // aggregate types are passed by pointer if (!inputarg.ispointer()) inputarg = value_to_pointer(ctx, inputarg); - arg = maybe_bitcast(ctx, decay_derived(data_pointer(ctx, inputarg)), + arg = maybe_bitcast(ctx, decay_derived(ctx, data_pointer(ctx, inputarg)), T->getPointerTo()); } else { @@ -4996,12 +5005,13 @@ static Function* gen_cfun_wrapper( ctx.builder.CreateICmpEQ( ctx.builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), ConstantInt::get(T_int8, 0)), - decay_derived(ctx.builder.CreateBitCast(result, T_pjlvalue)), - decay_derived(box)); + decay_derived(ctx, ctx.builder.CreateBitCast(result, T_pjlvalue)), + decay_derived(ctx, box)); retval = mark_julia_slot(derived, astrt, tindex, tbaa_stack); + assert(box->getType() == T_prjlvalue); retval.Vboxed = box; break; } @@ -5226,7 +5236,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con assert(jl_datatype_size(output_type) == sizeof(void*) * 4); Value *strct = emit_allocobj(ctx, jl_datatype_size(output_type), literal_pointer_val(ctx, (jl_value_t*)output_type)); - Value *derived_strct = emit_bitcast(ctx, decay_derived(strct), T_psize); + Value *derived_strct = emit_bitcast(ctx, decay_derived(ctx, strct), T_psize); MDNode *tbaa = best_tbaa(output_type); tbaa_decorate(tbaa, ctx.builder.CreateStore(F, derived_strct)); tbaa_decorate(tbaa, ctx.builder.CreateStore( @@ -5371,7 +5381,7 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret ty); } if (!isboxed) { - theArg = decay_derived(emit_bitcast(ctx, theArg, PointerType::get(lty, 0))); + theArg = decay_derived(ctx, emit_bitcast(ctx, theArg, PointerType::get(lty, 0))); if (!lty->isAggregateType()) // keep "aggregate" type values in place as pointers theArg = ctx.builder.CreateAlignedLoad(theArg, julia_alignment(ty)); } @@ -5412,6 +5422,7 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret ctx.builder.CreateExtractValue(call, 1), tbaa_stack); retval.Vboxed = ctx.builder.CreateExtractValue(call, 0); + assert(retval.Vboxed->getType() == T_prjlvalue); break; case jl_returninfo_t::Ghosts: retval = mark_julia_slot(NULL, jlretty, call, tbaa_stack); @@ -5530,7 +5541,7 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, String static void emit_sret_roots(jl_codectx_t &ctx, bool isptr, Value *Src, Type *T, Value *Shadow, unsigned count) { if (isptr) - Src = maybe_decay_tracked(Src); + Src = maybe_decay_tracked(ctx, Src); if (isptr && Src->getType()->getPointerElementType() != T) Src = ctx.builder.CreateBitCast(Src, T->getPointerTo(Src->getType()->getPointerAddressSpace())); unsigned emitted = TrackWithShadow(Src, T, isptr, Shadow, ctx.builder); @@ -6195,7 +6206,7 @@ static std::pair, jl_llvm_functions_t> emit_varinfo_assign(ctx, vi, tuple); } else { - restTuple = emit_jlcall(ctx, jltuple_func, maybe_decay_untracked(V_null), + restTuple = emit_jlcall(ctx, jltuple_func, V_rnull, vargs, ctx.nvargs, JLCALL_F_CC); jl_cgval_t tuple = mark_julia_type(ctx, restTuple, true, vi.value.typ); emit_varinfo_assign(ctx, vi, tuple); @@ -6206,7 +6217,7 @@ static std::pair, jl_llvm_functions_t> Function *F = prepare_call(jltuple_func); restTuple = ctx.builder.CreateCall(F, - { maybe_decay_untracked(V_null), + { V_rnull, ctx.builder.CreateInBoundsGEP(T_prjlvalue, argArray, ConstantInt::get(T_size, nreq - 1)), ctx.builder.CreateSub(argCount, @@ -6524,21 +6535,18 @@ static std::pair, jl_llvm_functions_t> Value *data, *tindex; if (retvalinfo.TIndex) { tindex = retvalinfo.TIndex; + data = V_rnull; if (retvalinfo.V == NULL) { // treat this as a simple Ghosts - data = maybe_decay_untracked(V_null); sret = NULL; } - else { - data = maybe_decay_untracked(V_null); - if (retvalinfo.Vboxed) { - // also need to account for the possibility the return object is boxed - // and avoid / skip copying it to the stack - isboxed_union = ctx.builder.CreateICmpNE( - ctx.builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), - ConstantInt::get(T_int8, 0)); - data = ctx.builder.CreateSelect(isboxed_union, retvalinfo.Vboxed, data); - } + else if (retvalinfo.Vboxed) { + // also need to account for the possibility the return object is boxed + // and avoid / skip copying it to the stack + isboxed_union = ctx.builder.CreateICmpNE( + ctx.builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)); + data = ctx.builder.CreateSelect(isboxed_union, retvalinfo.Vboxed, data); } } else { @@ -6546,7 +6554,7 @@ static std::pair, jl_llvm_functions_t> //assert(retvalinfo.isboxed); tindex = compute_tindex_unboxed(ctx, retvalinfo, jlrettype); tindex = ctx.builder.CreateOr(tindex, ConstantInt::get(T_int8, 0x80)); - data = maybe_decay_untracked(boxed(ctx, retvalinfo)); + data = boxed(ctx, retvalinfo); sret = NULL; } retval = UndefValue::get(retty); @@ -6785,7 +6793,7 @@ static std::pair, jl_llvm_functions_t> } else if (dest && val.typ != (jl_value_t*)jl_bottom_type) { assert(lty != T_prjlvalue); - (void)emit_unbox(ctx, lty, val, phiType, maybe_decay_tracked(dest)); + (void)emit_unbox(ctx, lty, val, phiType, maybe_decay_tracked(ctx, dest)); } } else { @@ -7340,6 +7348,7 @@ static void init_julia_llvm_env(Module *m) T_ppjlvalue = PointerType::get(T_pjlvalue, 0); T_pprjlvalue = PointerType::get(T_prjlvalue, 0); V_null = Constant::getNullValue(T_pjlvalue); + V_rnull = Constant::getNullValue(T_prjlvalue); std::vector ftargs(0); ftargs.push_back(T_prjlvalue); // function diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 47a7e79fc49ef..8877148addbcc 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -804,8 +804,8 @@ static jl_cgval_t emit_ifelse(jl_codectx_t &ctx, jl_cgval_t c, jl_cgval_t x, jl_ ifelse_tbaa = x.tbaa; } else { - x_ptr = decay_derived(x_ptr); - y_ptr = decay_derived(y_ptr); + x_ptr = decay_derived(ctx, x_ptr); + y_ptr = decay_derived(ctx, y_ptr); if (x_ptr->getType() != y_ptr->getType()) y_ptr = ctx.builder.CreateBitCast(y_ptr, x_ptr->getType()); ifelse_result = ctx.builder.CreateSelect(isfalse, y_ptr, x_ptr); @@ -861,6 +861,7 @@ static jl_cgval_t emit_ifelse(jl_codectx_t &ctx, jl_cgval_t c, jl_cgval_t x, jl_ if (!y_vboxed) y_vboxed = ConstantPointerNull::get(cast(x_vboxed->getType())); ret.Vboxed = ctx.builder.CreateSelect(isfalse, y_vboxed, x_vboxed); + assert(ret.Vboxed->getType() == T_prjlvalue); } return ret; } diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index d078d70c8d0d0..9b8e53eb5a3c7 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -600,12 +600,12 @@ Value *LateLowerGCFrame::MaybeExtractScalar(State &S, std::pair ValE std::vector LateLowerGCFrame::MaybeExtractVector(State &S, Value *BaseVec, Instruction *InsertBefore) { auto Numbers = NumberAllBase(S, BaseVec); std::vector V{Numbers.size()}; - Value *V_null = ConstantPointerNull::get(cast(T_prjlvalue)); + Value *V_rnull = ConstantPointerNull::get(cast(T_prjlvalue)); for (unsigned i = 0; i < V.size(); ++i) { if (Numbers[i] >= 0) V[i] = GetPtrForNumber(S, Numbers[i], InsertBefore); else - V[i] = V_null; + V[i] = V_rnull; } return V; }