Skip to content

Commit

Permalink
Merge pull request JuliaLang#24197 from JuliaLang/jn/codegen-union-la…
Browse files Browse the repository at this point in the history 
…yout

codegen: fix union layout alignment and padding
  • Loading branch information
@vtjnash
vtjnash committed Oct 19, 2017
2 parents a1c9549 + 43a6206 commit 40ee27a
Show file tree
Hide file tree
Showing 4 changed files with 177 additions and 119 deletions.
148 changes: 84 additions & 64 deletions src/cgutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,29 @@ static Value *julia_binding_gv(jl_codectx_t &ctx, jl_binding_t *b)

static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua_env, bool *isboxed);

static unsigned convert_struct_offset(Type *lty, unsigned byte_offset)
{
const DataLayout &DL =
#if JL_LLVM_VERSION >= 40000
jl_data_layout;
#else
jl_ExecutionEngine->getDataLayout();
#endif
const StructLayout *SL = DL.getStructLayout(cast<StructType>(lty));
return SL->getElementContainingOffset(byte_offset);
}

static unsigned convert_struct_offset(jl_codectx_t &ctx, Type *lty, unsigned byte_offset)
{
return convert_struct_offset(lty, byte_offset);
}

static Value *emit_struct_gep(jl_codectx_t &ctx, Type *lty, Value *base, unsigned byte_offset)
{
unsigned idx = convert_struct_offset(ctx, lty, byte_offset);
return ctx.builder.CreateConstInBoundsGEP2_32(lty, base, 0, idx);
}

extern "C" {
JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed)
{
Expand Down Expand Up @@ -484,7 +507,7 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox
return T_void;
if (!julia_struct_has_layout(jst, ua))
return NULL;
std::vector<Type*> latypes(ntypes);
std::vector<Type*> latypes(0);
bool isarray = true;
bool isvector = true;
jl_value_t *jlasttype = NULL;
Expand All @@ -508,32 +531,37 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox
fsz += 1;
}
Type *lty;
if (isptr)
if (isptr) {
lty = T_pjlvalue;
else if (ty == (jl_value_t*)jl_bool_type)
}
else if (ty == (jl_value_t*)jl_bool_type) {
lty = T_int8;
}
else if (jl_is_uniontype(ty)) {
// pick an Integer type size such that alignment will be correct
// and always end with an Int8 (selector byte)
lty = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (fsz - 1) / al);
std::vector<Type*> Elements(2);
Elements[0] = lty;
Elements[1] = T_int8;
Type *AlignmentType = IntegerType::get(jl_LLVMContext, 8 * al);
unsigned NumATy = (fsz - 1) / al;
unsigned remainder = (fsz - 1) % al;
while (NumATy--)
latypes.push_back(AlignmentType);
while (remainder--)
Elements.push_back(T_int8);
lty = StructType::get(jl_LLVMContext, Elements);
latypes.push_back(T_int8);
latypes.push_back(T_int8);
isarray = false;
allghost = false;
continue;
}
else
else {
lty = julia_type_to_llvm(ty);
}
if (lasttype != NULL && lasttype != lty)
isarray = false;
lasttype = lty;
if (type_is_ghost(lty))
lty = NoopType;
else
if (!type_is_ghost(lty)) {
allghost = false;
latypes[i] = lty;
latypes.push_back(lty);
}
}
Type *decl;
if (allghost) {
Expand All @@ -551,27 +579,16 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox
decl = ArrayType::get(lasttype, ntypes);
}
else {
#if 0 // stress-test code that tries to assume julia-index == llvm-index
// (also requires change to emit_new_struct to not assume 0 == 0)
if (!isTuple && latypes.size() > 1) {
Type *NoopType = ArrayType::get(T_int1, 0);
latypes.insert(latypes.begin(), NoopType);
}
#endif
decl = StructType::get(jl_LLVMContext, latypes);
}
jst->struct_decl = decl;
#ifndef JL_NDEBUG
// If LLVM and Julia disagree about alignment, much trouble ensues, so check it!
if (jst->layout) {
const DataLayout &DL =
#if JL_LLVM_VERSION >= 40000
jl_data_layout;
#else
jl_ExecutionEngine->getDataLayout();
#endif
unsigned llvm_alignment = DL.getABITypeAlignment((Type*)jst->struct_decl);
unsigned julia_alignment = jl_datatype_align(jst);
// Check that the alignment adheres to the heap alignment.
assert(julia_alignment <= JL_HEAP_ALIGNMENT);
// TODO: Fix alignment calculation in LLVM, as well as in the GC and the struct declaration
if (llvm_alignment <= JL_HEAP_ALIGNMENT)
assert(julia_alignment == llvm_alignment);
}
#endif
return decl;
}
// TODO: enable this (with tests):
Expand Down Expand Up @@ -1202,7 +1219,7 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
else
data = ptr;
if (idx_0based)
data = ctx.builder.CreateGEP(data, idx_0based);
data = ctx.builder.CreateGEP(elty, data, idx_0based);
Value *elt;
// TODO: can only lazy load if we can create a gc root for ptr for the lifetime of elt
//if (elty->isAggregateType() && tbaa == tbaa_immut && !alignment) { // can lazy load on demand, no copy needed
Expand Down Expand Up @@ -1256,8 +1273,9 @@ static void typed_store(jl_codectx_t &ctx,
} else {
data = ptr;
}
Instruction *store = ctx.builder.CreateAlignedStore(r, idx_0based ? ctx.builder.CreateGEP(data,
idx_0based) : data, isboxed ? alignment : julia_alignment(jltype, alignment));
if (idx_0based)
data = ctx.builder.CreateGEP(r->getType(), data, idx_0based);
Instruction *store = ctx.builder.CreateAlignedStore(r, data, isboxed ? alignment : julia_alignment(jltype, alignment));
if (tbaa)
tbaa_decorate(tbaa, store);
}
Expand Down Expand Up @@ -1394,7 +1412,7 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx,
if (!stt->mutabl) {
// just compute the pointer and let user load it when necessary
Type *fty = julia_type_to_llvm(jt);
Value *addr = ctx.builder.CreateGEP(emit_bitcast(ctx, ptr, PointerType::get(fty,0)), idx);
Value *addr = ctx.builder.CreateGEP(fty, emit_bitcast(ctx, ptr, PointerType::get(fty, 0)), idx);
*ret = mark_julia_slot(addr, jt, NULL, strct.tbaa);
ret->isimmutable = strct.isimmutable;
return true;
Expand Down Expand Up @@ -1447,35 +1465,42 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
return ghostValue(jfty);
Value *fldv = NULL;
if (strct.ispointer()) {
Value *addr = decay_derived(data_pointer(ctx, strct));
Value *staddr = decay_derived(data_pointer(ctx, strct));
bool isboxed;
Type *lt = julia_type_to_llvm((jl_value_t*)jt, &isboxed);
size_t byte_offset = jl_field_offset(jt, idx);
Value *addr;
if (isboxed) {
size_t byte_offset = jl_field_offset(jt, idx);
// byte_offset == 0 is an important special case here, e.g.
// for single field wrapper types. Introducing the bitcast
// can pessimize mem2reg
if (byte_offset > 0) {
addr = ctx.builder.CreateGEP(
emit_bitcast(ctx, addr, T_pint8),
emit_bitcast(ctx, staddr, T_pint8),
ConstantInt::get(T_size, byte_offset));
}
else {
addr = staddr;
}
}
else {
if (VectorType *vlt = dyn_cast<VectorType>(lt)) {
// doesn't have the struct wrapper, so this must have been a VecElement
// cast to the element type so that it can be addressed with GEP
lt = vlt->getElementType();
Value *ptr = emit_bitcast(ctx, addr, lt->getPointerTo());
staddr = emit_bitcast(ctx, staddr, lt->getPointerTo());
Value *llvm_idx = ConstantInt::get(T_size, idx);
addr = ctx.builder.CreateGEP(lt, ptr, llvm_idx);
addr = ctx.builder.CreateGEP(lt, staddr, llvm_idx);
}
else if (lt->isSingleValueType()) {
addr = emit_bitcast(ctx, addr, lt->getPointerTo());
addr = emit_bitcast(ctx, staddr, lt->getPointerTo());
}
else {
Value *ptr = emit_bitcast(ctx, addr, lt->getPointerTo());
addr = ctx.builder.CreateStructGEP(lt, ptr, idx);
staddr = emit_bitcast(ctx, staddr, lt->getPointerTo());
if (isa<StructType>(lt))
addr = emit_struct_gep(ctx, lt, staddr, byte_offset);
else
addr = ctx.builder.CreateConstGEP2_32(lt, staddr, 0, idx);
}
}
unsigned align = jl_field_align(jt, idx);
Expand All @@ -1492,11 +1517,11 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st
}
else if (jl_is_uniontype(jfty)) {
int fsz = jl_field_size(jt, idx);
Value *ptindex = ctx.builder.CreateGEP(T_int8, emit_bitcast(ctx, addr, T_pint8), ConstantInt::get(T_size, fsz - 1));
Value *ptindex = emit_struct_gep(ctx, lt, staddr, byte_offset + fsz - 1);
Value *tindex = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1), ctx.builder.CreateLoad(T_int8, ptindex));
bool isimmutable = strct.isimmutable;
if (jt->mutabl) {
// move value to an immutable stack slot
// move value to an immutable stack slot (excluding tindex)
Type *AT = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * align), (fsz + align - 2) / align);
AllocaInst *lv = emit_static_alloca(ctx, AT);
if (align > 1)
Expand Down Expand Up @@ -1884,7 +1909,7 @@ static jl_value_t *static_constant_instance(Constant *constant, jl_value_t *jt)
if (cpn != NULL) {
assert(jl_is_cpointer_type(jt));
uint64_t val = 0;
return jl_new_bits(jt,&val);
return jl_new_bits(jt, &val);
}

// issue #8464
Expand Down Expand Up @@ -1913,8 +1938,8 @@ static jl_value_t *static_constant_instance(Constant *constant, jl_value_t *jt)

jl_value_t **tupleargs;
JL_GC_PUSHARGS(tupleargs, nargs);
for(size_t i=0; i < nargs; i++) {
tupleargs[i] = static_constant_instance(constant->getAggregateElement(i), jl_tparam(jt,i));
for (size_t i = 0; i < nargs; i++) {
tupleargs[i] = static_constant_instance(constant->getAggregateElement(i), jl_tparam(jt, i));
}
jl_value_t *tpl = jl_f_tuple(NULL, tupleargs, nargs);
JL_GC_POP();
Expand Down Expand Up @@ -2306,7 +2331,7 @@ static void emit_setfield(jl_codectx_t &ctx,
if (byte_offset > 0) {
addr = ctx.builder.CreateGEP(
emit_bitcast(ctx, decay_derived(addr), T_pint8),
ConstantInt::get(T_size, byte_offset));
ConstantInt::get(T_size, byte_offset)); // TODO: use emit_struct_gep
}
jl_value_t *jfty = jl_svecref(sty->types, idx0);
if (jl_field_isptr(sty, idx0)) {
Expand Down Expand Up @@ -2375,33 +2400,30 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
jl_value_t *jtype = jl_svecref(sty->types, i);
const jl_cgval_t &fval_info = argv[i];
emit_typecheck(ctx, fval_info, jtype, "new");
Type *fty;
if (type_is_ghost(lt))
continue;
else if (jl_is_vecelement_type(ty))
fty = lt;
else
fty = cast<CompositeType>(lt)->getTypeAtIndex(i);
Type *fty = julia_type_to_llvm(jtype);
if (type_is_ghost(fty))
continue;
Value *dest = NULL;
unsigned offs = jl_field_offset(sty, i);
unsigned llvm_idx = (i > 0 && isa<StructType>(lt)) ? convert_struct_offset(ctx, lt, offs) : i;
if (!init_as_value) {
// avoid unboxing the argument explicitly
// and use memcpy instead
dest = ctx.builder.CreateConstInBoundsGEP2_32(lt, strct, 0, i);
dest = ctx.builder.CreateConstInBoundsGEP2_32(lt, strct, 0, llvm_idx);
}
Value *fval = NULL;
if (jl_is_uniontype(jtype)) {
assert(!init_as_value && "unimplemented");
StructType *lt_i = cast<StructType>(fty);
// compute tindex from rhs
jl_cgval_t rhs_union = convert_julia_type(ctx, fval_info, jtype);
if (rhs_union.typ == jl_bottom_type)
return jl_cgval_t();
Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jtype);
tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1));
Value *ptindex = ctx.builder.CreateConstInBoundsGEP2_32(
lt_i, dest, 0, lt_i->getNumElements() - 1);
int fsz = jl_field_size(sty, i);
Value *ptindex = emit_struct_gep(ctx, lt, strct, offs + fsz - 1);
ctx.builder.CreateStore(tindex, ptindex);
if (!rhs_union.isghost)
emit_unionmove(ctx, dest, fval_info, NULL, false, NULL);
Expand All @@ -2416,7 +2438,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
if (init_as_value) {
assert(fval);
if (lt->isVectorTy())
strct = ctx.builder.CreateInsertElement(strct, fval, ConstantInt::get(T_int32, i));
strct = ctx.builder.CreateInsertElement(strct, fval, ConstantInt::get(T_int32, llvm_idx));
else if (jl_is_vecelement_type(ty))
strct = fval; // VecElement type comes unwrapped in LLVM.
else if (lt->isAggregateType())
Expand Down Expand Up @@ -2448,10 +2470,8 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
if (jl_field_isptr(sty, i)) {
tbaa_decorate(strctinfo.tbaa, ctx.builder.CreateStore(
ConstantPointerNull::get(cast<PointerType>(T_prjlvalue)),
emit_bitcast(ctx,
ctx.builder.CreateGEP(emit_bitcast(ctx, strct, T_pint8),
ConstantInt::get(T_size, jl_field_offset(sty, i))),
T_pprjlvalue)));
ctx.builder.CreateGEP(T_prjlvalue, emit_bitcast(ctx, strct, T_pprjlvalue),
ConstantInt::get(T_size, jl_field_offset(sty, i) / sizeof(void*)))));
}
}
for (size_t i = nargs; i < nf; i++) {
Expand Down
41 changes: 21 additions & 20 deletions src/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,6 @@ int32_t jl_jlcall_api(const char *fname)

// constants
static Constant *V_null;
static Type *NoopType;
extern "C" {
JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed=NULL);
}
Expand Down Expand Up @@ -2133,8 +2132,9 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const

if (at->isAggregateType()) { // Struct or Array
assert(arg1.ispointer() && arg2.ispointer());
size_t sz = jl_datatype_size(arg1.typ);
if (sz > 512 && !((jl_datatype_t*)arg1.typ)->layout->haspadding) {
jl_datatype_t *sty = (jl_datatype_t*)arg1.typ;
size_t sz = jl_datatype_size(sty);
if (sz > 512 && !sty->layout->haspadding) {
Value *answer = ctx.builder.CreateCall(prepare_call(memcmp_derived_func),
{
maybe_bitcast(ctx, decay_derived(data_pointer(ctx, arg1)), T_pint8),
Expand All @@ -2147,22 +2147,26 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const
Type *atp = at->getPointerTo();
Value *varg1 = maybe_bitcast(ctx, decay_derived(data_pointer(ctx, arg1)), atp);
Value *varg2 = maybe_bitcast(ctx, decay_derived(data_pointer(ctx, arg2)), atp);
jl_svec_t *types = ((jl_datatype_t*)arg1.typ)->types;
jl_svec_t *types = sty->types;
Value *answer = ConstantInt::get(T_int1, 1);
for (size_t i = 0, l = jl_svec_len(types); i < l; i++) {
jl_value_t *fldty = jl_svecref(types, i);
Value *subAns, *fld1, *fld2;
fld1 = ctx.builder.CreateConstGEP2_32(at, varg1, 0, i);
fld2 = ctx.builder.CreateConstGEP2_32(at, varg2, 0, i);
Type *at_i = cast<GetElementPtrInst>(fld1)->getResultElementType();
if (type_is_ghost(at_i))
if (type_is_ghost(julia_type_to_llvm(fldty)))
continue;
unsigned byte_offset = jl_field_offset(sty, i);
Value *subAns, *fld1, *fld2;
if (isa<StructType>(at)) {
fld1 = emit_struct_gep(ctx, at, varg1, byte_offset);
fld2 = emit_struct_gep(ctx, at, varg2, byte_offset);
}
else {
fld1 = ctx.builder.CreateConstInBoundsGEP2_32(at, varg1, 0, i);
fld2 = ctx.builder.CreateConstInBoundsGEP2_32(at, varg2, 0, i);
}
if (jl_is_uniontype(fldty)) {
unsigned tindex_offset = cast<StructType>(at_i)->getNumElements() - 1;
Value *ptindex1 = ctx.builder.CreateConstInBoundsGEP2_32(
at_i, fld1, 0, tindex_offset);
Value *ptindex2 = ctx.builder.CreateConstInBoundsGEP2_32(
at_i, fld2, 0, tindex_offset);
unsigned tindex_offset = byte_offset + jl_field_size(sty, i) - 1;
Value *ptindex1 = emit_struct_gep(ctx, at, varg1, tindex_offset);
Value *ptindex2 = emit_struct_gep(ctx, at, varg2, tindex_offset);
Value *tindex1 = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1),
ctx.builder.CreateLoad(T_int8, ptindex1));
Value *tindex2 = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1),
Expand Down Expand Up @@ -2845,12 +2849,12 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
}
else {
size_t offs = jl_field_offset(stt, fieldidx);
Value *ptr = emit_bitcast(ctx, decay_derived(data_pointer(ctx, obj)), T_pint8);
Value *llvm_idx = ConstantInt::get(T_size, offs);
Value *ptr = emit_bitcast(ctx, decay_derived(data_pointer(ctx, obj)), T_pprjlvalue);
Value *llvm_idx = ConstantInt::get(T_size, offs / sizeof(void*));
Value *addr = ctx.builder.CreateGEP(ptr, llvm_idx);
// emit this using the same type as emit_getfield_knownidx
// so that LLVM may be able to load-load forward them and fold the result
Value *fldv = tbaa_decorate(obj.tbaa, ctx.builder.CreateLoad(T_prjlvalue, emit_bitcast(ctx, addr, T_pprjlvalue)));
Value *fldv = tbaa_decorate(obj.tbaa, ctx.builder.CreateLoad(T_prjlvalue, addr));
Value *isdef = ctx.builder.CreateIsNotNull(fldv);
*ret = mark_julia_type(ctx, isdef, false, jl_bool_type);
}
Expand Down Expand Up @@ -6011,9 +6015,6 @@ static void init_julia_llvm_env(Module *m)

auto T_pint8_derived = PointerType::get(T_int8, AddressSpace::Derived);

// This type is used to create undef Values for use in struct declarations to skip indices
NoopType = ArrayType::get(T_int1, 0);

// add needed base debugging definitions to our LLVM environment
DIBuilder dbuilder(*m);
DIFile *julia_h = dbuilder.createFile("julia.h","");
Expand Down
Loading

0 comments on commit 40ee27a

Please sign in to comment.