Implement function multi versioning in sysimg

* Implementing function cloning pass.
* Hook up debug info lookup and sysimg loading to the processor initialization API.

src/Makefile

@@ -52,7 +52,7 @@ LLVMLINK :=
 ifeq ($(JULIACODEGEN),LLVM)
 SRCS += codegen jitlayers disasm debuginfo llvm-simdloop llvm-ptls llvm-muladd \
  llvm-late-gc-lowering llvm-lower-handlers llvm-gc-invariant-verifier \
- llvm-propagate-addrspaces llvm-alloc-opt cgmemmgr
+ llvm-propagate-addrspaces llvm-multiversioning llvm-alloc-opt cgmemmgr
 FLAGS += -I$(shell $(LLVM_CONFIG_HOST) --includedir)
 LLVM_LIBS := all
 ifeq ($(USE_POLLY),1)
@@ -185,10 +185,12 @@ $(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,\
  intrinsics.cpp jitlayers.h intrinsics.h debuginfo.h codegen_shared.h cgutils.cpp ccall.cpp abi_*.cpp processor.h)
 $(BUILDDIR)/processor.o $(BUILDDIR)/processor.dbg.obj: $(addprefix $(SRCDIR)/,processor_*.cpp processor.h features_*.h)
 $(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h
-$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(SRCDIR)/debuginfo.h
+$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: \
+ $(addprefix $(SRCDIR)/,debuginfo.h processor.h)
 $(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/debuginfo.h $(SRCDIR)/processor.h
 $(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h
 $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c
+$(BUILDDIR)/staticdata.o $(BUILDDIR)/staticdata.dbg.obj: $(SRCDIR)/processor.h
 $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc.h
 $(BUILDDIR)/gc-debug.o $(BUILDDIR)/gc-debug.dbg.obj: $(SRCDIR)/gc.h
 $(BUILDDIR)/gc-pages.o $(BUILDDIR)/gc-pages.dbg.obj: $(SRCDIR)/gc.h

src/anticodegen.c

@@ -36,10 +36,10 @@ int jl_getFunctionInfo(jl_frame_t **frames, uintptr_t pointer, int skipC, int no
  return 0;
 }
 
-void jl_register_fptrs(uint64_t sysimage_base, const char *base, const int32_t *offsets,
+void jl_register_fptrs(uint64_t sysimage_base, const struct _jl_sysimg_fptrs_t *fptrs,
  jl_method_instance_t **linfos, size_t n)
 {
- (void)sysimage_base; (void)base; (void)offsets; (void)linfos; (void)n;
+ (void)sysimage_base; (void)fptrs; (void)linfos; (void)n;
 }
 
 void jl_compile_linfo(jl_method_instance_t *li) { }

src/debuginfo.cpp

@@ -37,6 +37,7 @@ using llvm_file_magic = sys::fs::file_magic;
 #if defined(_OS_LINUX_)
 # include <link.h>
 #endif
+#include "processor.h"
 
 #include <string>
 #include <sstream>
@@ -706,20 +707,14 @@ openDebugInfo(StringRef debuginfopath, const debug_link_info &info)
 }
 
 static uint64_t jl_sysimage_base;
-static const char *sysimg_fvars_base = nullptr;
-static const int32_t *sysimg_fvars_offsets;
+static jl_sysimg_fptrs_t sysimg_fptrs;
 static jl_method_instance_t **sysimg_fvars_linfo;
 static size_t sysimg_fvars_n;
-static const void *sysimg_fvars(size_t idx)
-{
- return sysimg_fvars_base + sysimg_fvars_offsets[idx];
-}
-void jl_register_fptrs(uint64_t sysimage_base, const char *base, const int32_t *offsets,
+void jl_register_fptrs(uint64_t sysimage_base, const jl_sysimg_fptrs_t *fptrs,
  jl_method_instance_t **linfos, size_t n)
 {
  jl_sysimage_base = (uintptr_t)sysimage_base;
- sysimg_fvars_base = base;
- sysimg_fvars_offsets = offsets;
+ sysimg_fptrs = *fptrs;
  sysimg_fvars_linfo = linfos;
  sysimg_fvars_n = n;
 }
@@ -738,7 +733,7 @@ static void get_function_name_and_base(const object::ObjectFile *object, bool in
  int64_t slide, bool untrusted_dladdr)
 {
  // Assume we only need base address for sysimg for now
- if (!insysimage || !sysimg_fvars_base)
+ if (!insysimage || !sysimg_fptrs.base)
  saddr = nullptr;
  bool needs_saddr = saddr && (!*saddr || untrusted_dladdr);
  bool needs_name = name && (!*name || untrusted_dladdr);
@@ -1091,9 +1086,17 @@ static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skip
  return 1;
  }
  frame0->fromC = !isSysImg;
- if (isSysImg && sysimg_fvars_base && saddr) {
+ if (isSysImg && sysimg_fptrs.base && saddr) {
+ intptr_t diff = (uintptr_t)saddr - (uintptr_t)sysimg_fptrs.base;
+ for (size_t i = 0; i < sysimg_fptrs.nclones; i++) {
+ if (diff == sysimg_fptrs.clone_offsets[i]) {
+ uint32_t idx = sysimg_fptrs.clone_idxs[i] & jl_sysimg_val_mask;
+ frame0->linfo = sysimg_fvars_linfo[idx];
+ break;
+ }
+ }
  for (size_t i = 0; i < sysimg_fvars_n; i++) {
- if (saddr == sysimg_fvars(i)) {
+ if (diff == sysimg_fptrs.offsets[i]) {
  frame0->linfo = sysimg_fvars_linfo[i];
  break;
  }

src/jitlayers.cpp

@@ -97,7 +97,7 @@ void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM)
 
 // this defines the set of optimization passes defined for Julia at various optimization levels.
 // it assumes that the TLI and TTI wrapper passes have already been added.
-void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level)
+void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump_native)
 {
 #ifdef JL_DEBUG_BUILD
  PM->add(createGCInvariantVerifierPass(true));
@@ -133,6 +133,8 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level)
  PM->add(createLateLowerGCFramePass());
  PM->add(createLowerPTLSPass(imaging_mode));
 #endif
+ if (dump_native)
+ PM->add(createMultiVersioningPass());
  return;
  }
  PM->add(createPropagateJuliaAddrspaces());
@@ -172,6 +174,8 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level)
  PM->add(createAllocOptPass());
 #endif
  PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl.
+ if (dump_native)
+ PM->add(createMultiVersioningPass());
  PM->add(createSROAPass()); // Break up aggregate allocas
  PM->add(createInstructionCombiningPass()); // Cleanup for scalarrepl.
  PM->add(createJumpThreadingPass()); // Thread jumps.
@@ -1022,20 +1026,18 @@ void jl_add_to_shadow(Module *m)
 
 static void emit_offset_table(Module *mod, const std::vector<GlobalValue*> &vars, StringRef name)
 {
+ // Emit a global variable with all the variable addresses.
+ // The cloning pass will convert them into offsets.
  assert(!vars.empty());
- addComdat(GlobalAlias::create(GlobalVariable::ExternalLinkage, name + "_base", vars[0]));
- auto vbase = ConstantExpr::getPtrToInt(vars[0], T_size);
  size_t nvars = vars.size();
- std::vector<Constant*> offsets(nvars);
- for (size_t i = 0; i < nvars; i++) {
- auto ptrdiff = ConstantExpr::getSub(ConstantExpr::getPtrToInt(vars[i], T_size), vbase);
- offsets[i] = sizeof(void*) == 8 ? ConstantExpr::getTrunc(ptrdiff, T_uint32) : ptrdiff;
- }
- ArrayType *vars_type = ArrayType::get(T_uint32, nvars);
- addComdat(new GlobalVariable(*mod, vars_type, true,
- GlobalVariable::ExternalLinkage,
- ConstantArray::get(vars_type, ArrayRef<Constant*>(offsets)),
- name + "_offsets"));
+ std::vector<Constant*> addrs(nvars);
+ for (size_t i = 0; i < nvars; i++)
+ addrs[i] = ConstantExpr::getBitCast(vars[i], T_psize);
+ ArrayType *vars_type = ArrayType::get(T_psize, nvars);
+ new GlobalVariable(*mod, vars_type, true,
+ GlobalVariable::ExternalLinkage,
+ ConstantArray::get(vars_type, addrs),
+ name);
 }
 
 
@@ -1144,7 +1146,7 @@ void jl_dump_native(const char *bc_fname, const char *unopt_bc_fname, const char
  }
 
  if (bc_fname || obj_fname)
- addOptimizationPasses(&PM, jl_options.opt_level);
+ addOptimizationPasses(&PM, jl_options.opt_level, true);
 
  if (bc_fname) {
  // call output handler directly to avoid special case handling of `-` filename

src/jitlayers.h

@@ -43,7 +43,7 @@ extern size_t jltls_offset_idx;
 typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing
 
 void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM);
-void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level);
+void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump_native=false);
 void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL);
 GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name,
  uintptr_t init, size_t &idx);
@@ -191,6 +191,7 @@ Pass *createLateLowerGCFramePass();
 Pass *createLowerExcHandlersPass();
 Pass *createGCInvariantVerifierPass(bool Strong);
 Pass *createPropagateJuliaAddrspaces();
+Pass *createMultiVersioningPass();
 Pass *createAllocOptPass();
 // Whether the Function is an llvm or julia intrinsic.
 static inline bool isIntrinsicFunction(Function *F)

src/julia_internal.h

@@ -1001,7 +1001,9 @@ extern jl_sym_t *nospecialize_sym;
 extern jl_sym_t *boundscheck_sym;
 extern jl_sym_t *gc_preserve_begin_sym; extern jl_sym_t *gc_preserve_end_sym;
 
-void jl_register_fptrs(uint64_t sysimage_base, const char *base, const int32_t *offsets,
+struct _jl_sysimg_fptrs_t;
+
+void jl_register_fptrs(uint64_t sysimage_base, const struct _jl_sysimg_fptrs_t *fptrs,
  jl_method_instance_t **linfos, size_t n);
 
 extern arraylist_t partial_inst;