Cleanup lower PTLS pass

* Branch on whether the pass is running on sysimg or jit rather than
  whether we are in imaging mode or not.

  This eliminate the need to setup hooks in the JIT to handle sysimg code.

* Move the set up of system image global variables completely to the pass.

  Decouple it from the gvars array and codegen no longer need to know
  how the tls access will be lowered.

* Make the tls getter function an intrinsic

  Making it easier to catch when the function is used incorrectly.
  Also make it impossible for the JIT to use a wrong function address.

Note that with these changes, the lower PTLS pass isn't an pure optimization anymore.
It is now required to generate valid code to access the TLS states.

src/codegen.cpp

@@ -280,13 +280,6 @@ static GlobalVariable *jldll_var;
 #endif //_OS_WINDOWS_
 
 static Function *jltls_states_func;
-#ifndef JULIA_ENABLE_THREADING
-static GlobalVariable *jltls_states_var;
-#else
-// Imaging mode only
-size_t jltls_states_func_idx = 0;
-size_t jltls_offset_idx = 0;
-#endif
 
 // important functions
 static Function *jlnew_func;
@@ -6127,43 +6120,9 @@ static void init_julia_llvm_env(Module *m)
  add_named_global(jldll_var, &jl_dl_handle);
 #endif
 
-#ifndef JULIA_ENABLE_THREADING
- // For non-threading, we use the address of the global variable directly
- jltls_states_var =
- new GlobalVariable(*m, T_ppjlvalue,
- false, GlobalVariable::ExternalLinkage,
- NULL, "jl_tls_states");
- add_named_global(jltls_states_var, &jl_tls_states);
- // placeholder function for keeping track of the end of the gcframe
- jltls_states_func = Function::Create(FunctionType::get(jltls_states_var->getType(), false),
- Function::ExternalLinkage,
- "jl_get_ptls_states", m);
- add_named_global(jltls_states_func, (void*)NULL, /*dllimport*/false);
-#else
- // For threading, we emit a call to the getter function.
- // In non-imaging mode, (i.e. the code will not be saved to disk), we
- // use the address of the actual getter function directly
- // (`jl_tls_states_cb` returned by `jl_get_ptls_states_getter()`)
- // (Alternatively if we know how to generate the tls address directly
- // we will inline the assembly, see `finalize_gc_frame(Function*)`)
- // In imaging mode, we emit the function address as a load of a static
- // variable to be filled (in `dump.c`) at initialization time of the sysimg.
- // This way we can by pass the extra indirection in `jl_get_ptls_states`
- // since we don't know which getter function to use ahead of time.
  jltls_states_func = Function::Create(FunctionType::get(PointerType::get(T_ppjlvalue, 0), false),
- Function::ExternalLinkage,
- "jl_get_ptls_states", m);
- add_named_global(jltls_states_func, jl_get_ptls_states_getter());
- if (imaging_mode) {
- PointerType *pfunctype = jltls_states_func->getFunctionType()->getPointerTo();
- jl_emit_sysimg_slot(m, pfunctype, "jl_get_ptls_states.ptr",
- (uintptr_t)jl_get_ptls_states_getter(),
- jltls_states_func_idx);
- jl_emit_sysimg_slot(m, T_size, "jl_tls_offset.val",
- (uintptr_t)(jl_tls_offset == -1 ? 0 : jl_tls_offset),
- jltls_offset_idx);
- }
-#endif
+ Function::ExternalLinkage, "julia.ptls_states");
+ add_named_global(jltls_states_func, (void*)NULL, /*dllimport*/false);
 
  std::vector<Type*> args1(0);
  args1.push_back(T_pint8);

src/codegen_shared.h

@@ -1,3 +1,7 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#include <llvm/Support/Debug.h>
+
 enum AddressSpace {
  Generic = 0,
  Tracked = 10, Derived = 11, CalleeRooted = 12,

src/jitlayers.cpp

@@ -117,7 +117,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump
  PM->add(createLowerExcHandlersPass());
  PM->add(createGCInvariantVerifierPass(false));
  PM->add(createLateLowerGCFramePass());
- PM->add(createLowerPTLSPass(imaging_mode));
+ PM->add(createLowerPTLSPass(dump_native));
  PM->add(createBarrierNoopPass());
 #endif
  PM->add(createMemCpyOptPass()); // Remove memcpy / form memset
@@ -131,7 +131,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump
  PM->add(createLowerExcHandlersPass());
  PM->add(createGCInvariantVerifierPass(false));
  PM->add(createLateLowerGCFramePass());
- PM->add(createLowerPTLSPass(imaging_mode));
+ PM->add(createLowerPTLSPass(dump_native));
 #endif
  if (dump_native)
  PM->add(createMultiVersioningPass());
@@ -156,7 +156,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump
  PM->add(createLateLowerGCFramePass());
  // Remove dead use of ptls
  PM->add(createDeadCodeEliminationPass());
- PM->add(createLowerPTLSPass(imaging_mode));
+ PM->add(createLowerPTLSPass(dump_native));
 #endif
 
  PM->add(createMemCpyOptPass());
@@ -257,7 +257,7 @@ void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump
  PM->add(createLateLowerGCFramePass());
  // Remove dead use of ptls
  PM->add(createDeadCodeEliminationPass());
- PM->add(createLowerPTLSPass(imaging_mode));
+ PM->add(createLowerPTLSPass(dump_native));
 #endif
  PM->add(createCombineMulAddPass());
 }
@@ -975,29 +975,6 @@ void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit)
  return slot;
 }
 
-// Emit a slot in the system image to be filled at sysimg init time.
-// Returns the global var. Fill `idx` with 1-base index in the sysimg gv.
-// Use as an optimization for runtime constant addresses to have one less
-// load. (Used only by threading).
-GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name,
- uintptr_t init, size_t &idx)
-{
- assert(imaging_mode);
- // This is **NOT** a external variable or a normal global variable
- // This is a special internal global slot with a special index
- // in the global variable table.
- GlobalVariable *gv = new GlobalVariable(*m, typ, false,
- GlobalVariable::InternalLinkage,
- Constant::getNullValue(typ), name);
- addComdat(gv);
- // make the pointer valid for this session
- auto p = new uintptr_t(init);
- jl_ExecutionEngine->addGlobalMapping(gv, (void*)p);
- jl_sysimg_gvars.push_back(gv);
- idx = jl_sysimg_gvars.size();
- return gv;
-}
-
 void* jl_get_globalvar(GlobalVariable *gv)
 {
  void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable(gv);
@@ -1051,20 +1028,6 @@ static void jl_gen_llvm_globaldata(Module *mod, const char *sysimg_data, size_t
  GlobalVariable::ExternalLinkage,
  ConstantInt::get(T_size, globalUnique+1),
  "jl_globalUnique"));
-#ifdef JULIA_ENABLE_THREADING
- addComdat(new GlobalVariable(*mod,
- T_size,
- true,
- GlobalVariable::ExternalLinkage,
- ConstantInt::get(T_size, jltls_states_func_idx),
- "jl_ptls_states_getter_idx"));
- addComdat(new GlobalVariable(*mod,
- T_size,
- true,
- GlobalVariable::ExternalLinkage,
- ConstantInt::get(T_size, jltls_offset_idx),
- "jl_tls_offset_idx"));
-#endif
 
  // reflect the address of the jl_RTLD_DEFAULT_handle variable
  // back to the caller, so that we can check for consistency issues

src/jitlayers.h

@@ -35,18 +35,11 @@ extern Function *juliapersonality_func;
 #endif
 
 
-#ifdef JULIA_ENABLE_THREADING
-extern size_t jltls_states_func_idx;
-extern size_t jltls_offset_idx;
-#endif
-
 typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing
 
 void addTargetPasses(legacy::PassManagerBase *PM, TargetMachine *TM);
 void addOptimizationPasses(legacy::PassManagerBase *PM, int opt_level, bool dump_native=false);
 void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL);
-GlobalVariable *jl_emit_sysimg_slot(Module *m, Type *typ, const char *name,
- uintptr_t init, size_t &idx);
 void* jl_get_globalvar(GlobalVariable *gv);
 GlobalVariable *jl_get_global_for(const char *cname, void *addr, Module *M);
 void jl_add_to_shadow(Module *m);

src/llvm-late-gc-lowering.cpp

@@ -1532,7 +1532,7 @@ static void addRetNoAlias(Function *F)
 }
 
 bool LateLowerGCFrame::DefineFunctions(Module &M) {
- ptls_getter = M.getFunction("jl_get_ptls_states");
+ ptls_getter = M.getFunction("julia.ptls_states");
  gc_flush_func = M.getFunction("julia.gcroot_flush");
  gc_preserve_begin_func = M.getFunction("llvm.julia.gc_preserve_begin");
  gc_preserve_end_func = M.getFunction("llvm.julia.gc_preserve_end");