Allow LLVM to optimize GC write barriers for new objects (JuliaLang#3…

…6992)

Make alignment of tag more obvious for LLVM and
add a few optimization passes after GC lowering to let LLVM
delete some write barriers.

This fixes a regression from LLVM < 5 where the pass ordering makes this
happen automatically.

src/aotcompile.cpp

@@ -627,11 +627,14 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
  PM->add(createBarrierNoopPass());
  PM->add(createLowerExcHandlersPass());
  PM->add(createGCInvariantVerifierPass(false));
+ PM->add(createRemoveNIPass());
  PM->add(createLateLowerGCFramePass());
  PM->add(createFinalLowerGCPass());
  PM->add(createLowerPTLSPass(dump_native));
  }
- PM->add(createRemoveNIPass());
+ else {
+ PM->add(createRemoveNIPass());
+ }
  PM->add(createLowerSimdLoopPass()); // Annotate loop marked with "loopinfo" as LLVM parallel loop
  if (dump_native)
  PM->add(createMultiVersioningPass());
@@ -751,15 +754,26 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level,
  PM->add(createBarrierNoopPass());
  PM->add(createLowerExcHandlersPass());
  PM->add(createGCInvariantVerifierPass(false));
+ // Needed **before** LateLowerGCFrame on LLVM < 12
+ // due to bug in `CreateAlignmentAssumption`.
+ PM->add(createRemoveNIPass());
  PM->add(createLateLowerGCFramePass());
  PM->add(createFinalLowerGCPass());
+ // We need these two passes and the instcombine below
+ // after GC lowering to let LLVM do some constant propagation on the tags.
+ // and remove some unnecessary write barrier checks.
+ PM->add(createGVNPass());
+ PM->add(createSCCPPass());
  // Remove dead use of ptls
  PM->add(createDeadCodeEliminationPass());
  PM->add(createLowerPTLSPass(dump_native));
+ PM->add(createInstructionCombiningPass());
  // Clean up write barrier and ptls lowering
  PM->add(createCFGSimplificationPass());
  }
- PM->add(createRemoveNIPass());
+ else {
+ PM->add(createRemoveNIPass());
+ }
  PM->add(createCombineMulAddPass());
  PM->add(createDivRemPairsPass());
 #if defined(JL_ASAN_ENABLED)

src/ccall.cpp

@@ -381,7 +381,10 @@ static Value *runtime_apply_type_env(jl_codectx_t &ctx, jl_value_t *ty)
  ctx.spvals_ptr,
  ConstantInt::get(T_size, sizeof(jl_svec_t) / sizeof(jl_value_t*)))
  };
- return ctx.builder.CreateCall(prepare_call(jlapplytype_func), makeArrayRef(args));
+ auto call = ctx.builder.CreateCall(prepare_call(jlapplytype_func), makeArrayRef(args));
+ call->addAttribute(AttributeList::ReturnIndex,
+ Attribute::getWithAlignment(jl_LLVMContext, Align(16)));
+ return call;
 }
 
 static const std::string make_errmsg(const char *fname, int n, const char *err)

src/cgutils.cpp

@@ -335,6 +335,11 @@ static unsigned julia_alignment(jl_value_t *jt)
  // Array always has this alignment
  return JL_SMALL_BYTE_ALIGNMENT;
  }
+ if (jt == (jl_value_t*)jl_datatype_type) {
+ // types are never allocated in julia code/on the stack
+ // and this is the guarantee we have for the GC bits
+ return 16;
+ }
  assert(jl_is_datatype(jt) && ((jl_datatype_t*)jt)->layout);
  unsigned alignment = jl_datatype_align(jt);
  if (alignment > JL_HEAP_ALIGNMENT)

src/codegen.cpp

@@ -659,10 +659,13 @@ static const auto jlapplytype_func = new JuliaFunction{
  "jl_instantiate_type_in_env",
  [](LLVMContext &C) { return FunctionType::get(T_prjlvalue,
  {T_pjlvalue, T_pjlvalue, T_pprjlvalue}, false); },
- [](LLVMContext &C) { return AttributeList::get(C,
+ [](LLVMContext &C) {
+ return AttributeList::get(C,
  AttributeSet(),
- Attributes(C, {Attribute::NonNull}),
- None); },
+ AttributeSet::get(C, makeArrayRef({Attribute::get(C, Attribute::NonNull),
+ Attribute::getWithAlignment(C, Align(16))})),
+ None);
+ },
 };
 static const auto jl_object_id__func = new JuliaFunction{
  "jl_object_id_",

src/llvm-late-gc-lowering.cpp

@@ -1133,6 +1133,14 @@ static bool isLoadFromConstGV(LoadInst *LI)
  return false;
 }
 
+static uint64_t getLoadValueAlign(LoadInst *LI)
+{
+ MDNode *md = LI->getMetadata(LLVMContext::MD_align);
+ if (!md)
+ return 1;
+ return mdconst::extract<ConstantInt>(md->getOperand(0))->getLimitedValue();
+}
+
 static bool LooksLikeFrameRef(Value *V) {
  if (isSpecialPtr(V->getType()))
  return false;
@@ -2106,11 +2114,54 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S) {
  });
  newI->takeName(CI);
 
+ // LLVM alignment/bit check is not happy about addrspacecast and refuse
+ // to remove write barrier because of it.
+ // We pretty much only load using `T_size` so try our best to strip
+ // as many cast as possible.
+#if JL_LLVM_VERSION >= 100000
+ auto tag = CI->getArgOperand(2)->stripPointerCastsAndAliases();
+#else
+ auto tag = CI->getArgOperand(2)->stripPointerCasts();
+#endif
+ if (auto C = dyn_cast<ConstantExpr>(tag)) {
+ if (C->getOpcode() == Instruction::IntToPtr) {
+ tag = C->getOperand(0);
+ }
+ }
+ else if (auto LI = dyn_cast<LoadInst>(tag)) {
+ // Make sure the load is correctly marked as aligned
+ // since LLVM might have removed them.
+ // We can't do this in general since the load might not be
+ // a type in other branches.
+ // However, it should be safe for us to do this on const globals
+ // which should be the important cases as well.
+ if (isLoadFromConstGV(LI) && getLoadValueAlign(LI) < 16) {
+ Type *T_int64 = Type::getInt64Ty(LI->getContext());
+ auto op = ConstantAsMetadata::get(ConstantInt::get(T_int64, 16));
+ LI->setMetadata(LLVMContext::MD_align,
+ MDNode::get(LI->getContext(), { op }));
+ }
+ }
+ // As a last resort, if we didn't manage to strip down the tag
+ // for LLVM, emit an alignment assumption.
+ auto tag_type = tag->getType();
+ if (tag_type->isPointerTy()) {
+ auto &DL = CI->getModule()->getDataLayout();
+#if JL_LLVM_VERSION >= 100000
+ auto align = tag->getPointerAlignment(DL).valueOrOne().value();
+#else
+ auto align = tag->getPointerAlignment(DL);
+#endif
+ if (align < 16) {
+ // On 5 <= LLVM < 12, it is illegal to call this on
+ // non-integral pointer. This relies on stripping the
+ // non-integralness from datalayout before this pass
+ builder.CreateAlignmentAssumption(DL, tag, 16);
+ }
+ }
  // Set the tag.
  StoreInst *store = builder.CreateAlignedStore(
- CI->getArgOperand(2),
- EmitTagPtr(builder, T_prjlvalue, newI),
- sizeof(size_t));
+ tag, EmitTagPtr(builder, tag_type, newI), sizeof(size_t));
  store->setOrdering(AtomicOrdering::Unordered);
  store->setMetadata(LLVMContext::MD_tbaa, tbaa_tag);
 

test/llvmpasses/late-lower-gc.ll

@@ -1,6 +1,6 @@
 ; RUN: opt -load libjulia%shlibext -LateLowerGCFrame -S %s | FileCheck %s
 
-@tag = external addrspace(10) global {}
+@tag = external addrspace(10) global {}, align 16
 
 declare void @boxed_simple({} addrspace(10)*, {} addrspace(10)*)
 declare {} addrspace(10)* @jl_box_int64(i64)