in codegen, use StructRet where appropriate

it seems that the llvm 3.3 return type legalizer may not be properly
handling large types. this is more efficient anyways.

fixes #8932, should fix #12163, probably fixes #7434

(cherry picked from commit 13c83db)

src/cgutils.cpp

@@ -930,6 +930,12 @@ static Value *mark_julia_type(Value *v, jl_value_t *jt)
  return v;
 }
 
+static bool deserves_sret(jl_value_t *dt, Type *T)
+{
+ assert(jl_is_datatype(dt));
+ return jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy();
+}
+
 // --- generating various field accessors ---
 
 static Value *emit_nthptr_addr(Value *v, ssize_t n)

src/codegen.cpp

@@ -462,6 +462,7 @@ typedef struct {
  std::string funcName;
  jl_sym_t *vaName; // name of vararg argument
  bool vaStack; // varargs stack-allocated
+ bool sret;
  int nReqArgs;
  int lineno;
  std::vector<bool> boundsCheck;
@@ -2527,14 +2528,20 @@ static Value *emit_call_function_object(jl_function_t *f, Value *theF, Value *th
  jl_value_t **args, size_t nargs,
  jl_codectx_t *ctx)
 {
- Value *result;
  if (f!=NULL && specialized && f->linfo!=NULL && f->linfo->specFunctionObject!=NULL) {
  // emit specialized call site
  Function *cf = (Function*)f->linfo->specFunctionObject;
  FunctionType *cft = cf->getFunctionType();
  size_t nfargs = cft->getNumParams();
  Value **argvals = (Value**) alloca(nfargs*sizeof(Value*));
+ bool sret = cf->hasStructRetAttr();
  unsigned idx = 0;
+ Value *result;
+ if (sret) {
+ result = emit_static_alloca(cft->getParamType(0)->getContainedType(0), ctx);
+ argvals[idx] = result;
+ idx++;
+ }
  for(size_t i=0; i < nargs; i++) {
  Type *at = cft->getParamType(idx);
  jl_value_t *jt = jl_nth_slot_type(f->linfo->specTypes,i);
@@ -2571,13 +2578,15 @@ static Value *emit_call_function_object(jl_function_t *f, Value *theF, Value *th
  idx++;
  }
  assert(idx == nfargs);
- result = builder.CreateCall(prepare_call(cf), ArrayRef<Value*>(&argvals[0],nfargs));
- result = mark_julia_type(emit_reg2mem(result, ctx), jl_ast_rettype(f->linfo, f->linfo->ast));
- }
- else {
- result = emit_jlcall(theFptr, theF, &args[1], nargs, ctx);
+ CallInst *call = builder.CreateCall(prepare_call(cf), ArrayRef<Value*>(&argvals[0], nfargs));
+ call->setAttributes(cf->getAttributes());
+ if (sret)
+ result = builder.CreateLoad(result);
+ else
+ result = call;
+ return mark_julia_type(emit_reg2mem(result, ctx), jl_ast_rettype(f->linfo, f->linfo->ast));
  }
- return result;
+ return emit_jlcall(theFptr, theF, &args[1], nargs, ctx);
 }
 
 static Value *emit_is_function(Value *x, jl_codectx_t *ctx)
@@ -3643,6 +3652,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t
  jl_codectx_t ctx;
  ctx.f = cw;
  ctx.linfo = lam;
+ ctx.sret = false;
  allocate_gc_frame(0, b0, &ctx);
 
  // Save the Function object reference
@@ -3656,15 +3666,17 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t
  lam->cFunctionList = list2;
 
  // See whether this function is specsig or jlcall
- bool specsig;
+ bool specsig, jlfunc_sret;
  Function *theFptr;
  if (lam->specFunctionObject != NULL) {
  theFptr = (Function*)lam->specFunctionObject;
  specsig = true;
+ jlfunc_sret = theFptr->hasStructRetAttr();
  }
  else {
  theFptr = (Function*)lam->functionObject;
  specsig = false;
+ jlfunc_sret = false;
  }
  assert(theFptr);
 
@@ -3676,7 +3688,17 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t
  if (sret)
  sretPtr = AI++;
 
+ Value *result;
  size_t FParamIndex = 0;
+ if (jlfunc_sret) {
+ if (sret)
+ result = sretPtr;
+ else
+ result = builder.CreateAlloca(theFptr->getFunctionType()->getParamType(0)->getContainedType(0));
+ args.push_back(result);
+ FParamIndex++;
+ }
+
  for (size_t i = 0; i < nargs; i++) {
  Value *val = AI++;
  jl_value_t *jargty = jl_nth_slot_type(lam->specTypes, i);
@@ -3749,7 +3771,13 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t
  // Create the call
  Value *r;
  if (specsig) {
- r = emit_reg2mem(builder.CreateCall(prepare_call(theFptr), ArrayRef<Value*>(args)), &ctx);
+ CallInst *call = builder.CreateCall(prepare_call(theFptr), ArrayRef<Value*>(args));
+ call->setAttributes(theFptr->getAttributes());
+ if (jlfunc_sret)
+ r = builder.CreateLoad(result);
+ else
+ r = call;
+ r = emit_reg2mem(r, &ctx);
  }
  else {
  r = emit_jlcall(theFptr, literal_pointer_val((jl_value_t*)ff), 0, nargs, &ctx);
@@ -3763,6 +3791,9 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t
  r = boxed(r, &ctx, jlrettype);
  }
  }
+ else if (sret && jlfunc_sret) {
+ // nothing to do
+ }
  else if (!type_is_ghost(crt)) {
  if (sret)
  prt = fargt_sig[0]->getContainedType(0); // sret is a PointerType
@@ -3826,7 +3857,7 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t
 }
 
 // generate a julia-callable function that calls f (AKA lam)
-static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f)
+static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret)
 {
  std::stringstream funcName;
  const std::string &fname = f->getName().str();
@@ -3840,23 +3871,31 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct
  funcName.str(), f->getParent());
  addComdat(w);
  Function::arg_iterator AI = w->arg_begin();
- AI++; //const Argument &fArg = *AI++;
+ /* const Argument &fArg = */ *AI++;
  Value *argArray = AI++;
- //const Argument &argCount = *AI++;
+ /* const Argument &argCount = *AI++; */
  BasicBlock *b0 = BasicBlock::Create(jl_LLVMContext, "top", w);
 
  builder.SetInsertPoint(b0);
  DebugLoc noDbg;
  builder.SetCurrentDebugLocation(noDbg);
 
  jl_codectx_t ctx;
+ ctx.f = w;
  ctx.linfo = lam;
+ ctx.sret = false;
  allocate_gc_frame(0, b0, &ctx);
 
  size_t nargs = jl_array_dim0(jl_lam_args(ast));
  size_t nfargs = f->getFunctionType()->getNumParams();
  Value **args = (Value**) alloca(nfargs*sizeof(Value*));
  unsigned idx = 0;
+ Value *result;
+ if (sret) {
+ result = builder.CreateAlloca(f->getFunctionType()->getParamType(0)->getContainedType(0));
+ args[idx] = result;
+ idx++;
+ }
  for(size_t i=0; i < nargs; i++) {
  jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i);
  Type *lty = julia_type_to_llvm(ty);
@@ -3878,9 +3917,19 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct
  }
  // TODO: consider pulling the function pointer out of fArg so these
  // wrappers can be reused for different functions of the same type.
- Value *r = builder.CreateCall(prepare_call(f), ArrayRef<Value*>(&args[0], nfargs));
- if (r->getType() != jl_pvalue_llvmt) {
- r = boxed(r, &ctx, jl_ast_rettype(lam, (jl_value_t*)ast));
+ CallInst *call = builder.CreateCall(prepare_call(f), ArrayRef<Value*>(&args[0], nfargs));
+ call->setAttributes(f->getAttributes());
+ Value *r;
+ if (sret || call->getType() != jl_pvalue_llvmt) {
+ jl_value_t *ty = jl_ast_rettype(lam, (jl_value_t*)ast);
+ if (sret)
+ r = builder.CreateLoad(result);
+ else
+ r = call;
+ r = boxed(r, &ctx, ty);
+ }
+ else {
+ r = call;
  }
 
  builder.CreateRet(r);
@@ -4033,8 +4082,15 @@ static Function *emit_function(jl_lambda_info_t *lam)
 #endif
  funcName << "_" << globalUnique++;
 
+ ctx.sret = false;
  if (specsig) { // assumes !va
  std::vector<Type*> fsig(0);
+ Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype));
+ if (rt != jl_pvalue_llvmt && rt != T_void && deserves_sret(jlrettype, rt)) {
+ ctx.sret = true;
+ fsig.push_back(rt->getPointerTo());
+ rt = T_void;
+ }
  for(size_t i=0; i < jl_nparams(lam->specTypes); i++) {
  Type *ty = julia_type_to_llvm(jl_tparam(lam->specTypes,i));
  if (type_is_ghost(ty)) {
@@ -4046,17 +4102,18 @@ static Function *emit_function(jl_lambda_info_t *lam)
  ty = PointerType::get(ty,0);
  fsig.push_back(ty);
  }
- Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype));
  f = Function::Create(FunctionType::get(rt, fsig, false),
  imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage,
  funcName.str(), m);
+ if (ctx.sret)
+ f->addAttribute(1, Attribute::StructRet);
  addComdat(f);
  if (lam->specFunctionObject == NULL) {
  lam->specFunctionObject = (void*)f;
  lam->specFunctionID = jl_assign_functionID(f);
  }
  if (lam->functionObject == NULL) {
- Function *fwrap = gen_jlcall_wrapper(lam, ast, f);
+ Function *fwrap = gen_jlcall_wrapper(lam, ast, f, ctx.sret);
  lam->functionObject = (void*)fwrap;
  lam->functionID = jl_assign_functionID(fwrap);
  }
@@ -4222,7 +4279,7 @@ static Function *emit_function(jl_lambda_info_t *lam)
  varinfo.dinfo = ctx.dbuilder->createParameterVariable(
  SP, // Scope (current function will be fill in later)
  argname->name, // Variable name
- i+1, // Argument number (1-based)
+ ctx.sret + i + 1, // Argument number (1-based)
  topfile, // File
  ctx.lineno == -1 ? 0 : ctx.lineno, // Line
  // Variable type
@@ -4237,15 +4294,15 @@ static Function *emit_function(jl_lambda_info_t *lam)
  julia_type_to_di(varinfo.declType,ctx.dbuilder,specsig), // Variable type
  false, // May be optimized out
  0, // Flags (TODO: Do we need any)
- i+1); // Argument number (1-based)
+ ctx.sret + i + 1); // Argument number (1-based)
 #endif
  }
  if (va) {
 #ifdef LLVM38
  ctx.vars[ctx.vaName].dinfo = ctx.dbuilder->createParameterVariable(
  SP, // Scope (current function will be fill in later)
  ctx.vaName->name, // Variable name
- nreq + 1, // Argument number (1-based)
+ ctx.sret + nreq + 1, // Argument number (1-based)
  topfile, // File
  ctx.lineno == -1 ? 0 : ctx.lineno, // Line (for now, use lineno of the function)
  julia_type_to_di(ctx.vars[ctx.vaName].declType,ctx.dbuilder,false));
@@ -4259,7 +4316,7 @@ static Function *emit_function(jl_lambda_info_t *lam)
  julia_type_to_di(ctx.vars[ctx.vaName].declType,ctx.dbuilder,false), // Variable type
  false, // May be optimized out
  0, // Flags (TODO: Do we need any)
- nreq + 1); // Argument number (1-based)
+ ctx.sret + nreq + 1); // Argument number (1-based)
 #endif
  }
  for(i=0; i < vinfoslen; i++) {
@@ -4490,6 +4547,8 @@ static Function *emit_function(jl_lambda_info_t *lam)
  // step 12. move args into local variables
  Function::arg_iterator AI = f->arg_begin();
  argIdx = 0;
+ if (ctx.sret)
+ AI++; // skip sret slot
  for(i=0; i < nreq; i++) {
  jl_sym_t *s = jl_decl_var(jl_cellref(largs,i));
  jl_varinfo_t &vi = ctx.vars[s];
@@ -4712,7 +4771,7 @@ static Function *emit_function(jl_lambda_info_t *lam)
  if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == return_sym) {
  jl_expr_t *ex = (jl_expr_t*)stmt;
  Value *retval;
- Type *retty = f->getReturnType();
+ Type *retty = ctx.sret ? f->getFunctionType()->getParamType(0)->getContainedType(0) : f->getReturnType();
  if (retty == jl_pvalue_llvmt) {
  retval = boxed(emit_expr(jl_exprarg(ex,0), &ctx, true),&ctx,expr_type(stmt,&ctx));
  }
@@ -4725,7 +4784,10 @@ static Function *emit_function(jl_lambda_info_t *lam)
  }
  if (do_malloc_log && lno != -1)
  mallocVisitLine(filename, lno);
- if (retty == T_void)
+
+ if (ctx.sret)
+ builder.CreateStore(retval, ctx.f->arg_begin());
+ if (type_is_ghost(retty) || ctx.sret)
  builder.CreateRetVoid();
  else
  builder.CreateRet(retval);
@@ -4811,8 +4873,15 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig)
  std::string funcName = lam->name->name;
  funcName = "julia_" + funcName;
  if (specsig) { // assumes !va
- jl_value_t *jlrettype = jl_ast_rettype(lam, (jl_value_t*)lam->ast);
  std::vector<Type*> fsig(0);
+ jl_value_t *jlrettype = jl_ast_rettype(lam, (jl_value_t*)lam->ast);
+ Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype));
+ bool sret = false;
+ if (rt != jl_pvalue_llvmt && rt != T_void && deserves_sret(jlrettype, rt)) {
+ sret = true;
+ fsig.push_back(rt->getPointerTo());
+ rt = T_void;
+ }
  for (size_t i=0; i < jl_nparams(lam->specTypes); i++) {
  Type *ty = julia_type_to_llvm(jl_tparam(lam->specTypes,i));
  if (type_is_ghost(ty))
@@ -4821,9 +4890,10 @@ extern "C" void jl_fptr_to_llvm(void *fptr, jl_lambda_info_t *lam, int specsig)
  ty = PointerType::get(ty,0);
  fsig.push_back(ty);
  }
- Type *rt = (jlrettype == (jl_value_t*)jl_void_type ? T_void : julia_type_to_llvm(jlrettype));
  Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName,
  shadow_module);
+ if (sret)
+ f->addAttribute(1, Attribute::StructRet);
 
  if (lam->specFunctionObject == NULL) {
  lam->specFunctionObject = (void*)f;

test/core.jl

@@ -3357,3 +3357,14 @@ end
 # issue #13183
 gg13183{X}(x::X...) = 1==0 ? gg13183(x, x) : 0
 @test gg13183(5) == 0
+
+# issue 8932 (llvm return type legalizer error)
+immutable Vec3_8932
+ x::Float32
+ y::Float32
+ z::Float32
+end
+f8932(a::Vec3_8932, b::Vec3_8932) = Vec3_8932(a.x % b.x, a.y % b.y, a.z % b.z)
+a8932 = Vec3_8932(1,1,1)
+b8932 = Vec3_8932(2,2,2)
+@test f8932(a8932, b8932) == Vec3_8932(1.0, 1.0, 1.0)