Skip to content

Commit

Permalink
Table-based dlsym
Browse files Browse the repository at this point in the history
  • Loading branch information
@pchintalapudi
pchintalapudi committed Mar 3, 2023
1 parent 2c7375c commit 6ab1862
Show file tree
Hide file tree
Showing 5 changed files with 214 additions and 89 deletions.
112 changes: 103 additions & 9 deletions src/aotcompile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,8 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
//Safe b/c context is locked by params
GlobalVariable *G = cast<GlobalVariable>(clone.getModuleUnlocked()->getNamedValue(global));
G->setInitializer(ConstantPointerNull::get(cast<PointerType>(G->getValueType())));
G->setLinkage(GlobalVariable::InternalLinkage);
G->setLinkage(GlobalValue::ExternalLinkage);
G->setVisibility(GlobalValue::HiddenVisibility);
data->jl_sysimg_gvars.push_back(G);
}
CreateNativeGlobals += gvars.size();
Expand All @@ -446,9 +447,9 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm
//Safe b/c context is locked by params
for (GlobalObject &G : clone.getModuleUnlocked()->global_objects()) {
if (!G.isDeclaration()) {
G.setLinkage(Function::InternalLinkage);
G.setLinkage(GlobalValue::ExternalLinkage);
G.setVisibility(GlobalValue::HiddenVisibility);
makeSafeName(G);
addComdat(&G);
#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_)
// Add unwind exception personalities to functions to handle async exceptions
if (Function *F = dyn_cast<Function>(&G))
Expand Down Expand Up @@ -514,6 +515,63 @@ static void injectCRTAlias(Module &M, StringRef name, StringRef alias, FunctionT

void multiversioning_preannotate(Module &M);

static GlobalVariable *emit_shard_table(Module &M, Type *T_size, Type *T_psize, unsigned threads) {
SmallVector<Constant *, 0> tables(sizeof(jl_image_shard_t) / sizeof(void *) * threads);
for (unsigned i = 0; i < threads; i++) {
auto suffix = "_" + std::to_string(i);
auto create_gv = [&](StringRef name, bool constant) {
auto gv = new GlobalVariable(M, T_size, constant,
GlobalValue::ExternalLinkage, nullptr, name + suffix);
gv->setVisibility(GlobalValue::HiddenVisibility);
return gv;
};
auto table = tables.data() + i * sizeof(jl_image_shard_t) / sizeof(void *);
table[offsetof(jl_image_shard_t, fvar_base) / sizeof(void*)] = create_gv("jl_fvar_base", false);
table[offsetof(jl_image_shard_t, fvar_offsets) / sizeof(void*)] = create_gv("jl_fvar_offsets", true);
table[offsetof(jl_image_shard_t, fvar_idxs) / sizeof(void*)] = create_gv("jl_fvar_idxs", true);
table[offsetof(jl_image_shard_t, gvar_base) / sizeof(void*)] = create_gv("jl_gvar_base", false);
table[offsetof(jl_image_shard_t, gvar_offsets) / sizeof(void*)] = create_gv("jl_gvar_offsets", true);
table[offsetof(jl_image_shard_t, gvar_idxs) / sizeof(void*)] = create_gv("jl_gvar_idxs", true);
table[offsetof(jl_image_shard_t, clone_slots) / sizeof(void*)] = create_gv("jl_clone_slots", true);
table[offsetof(jl_image_shard_t, clone_offsets) / sizeof(void*)] = create_gv("jl_clone_offsets", true);
table[offsetof(jl_image_shard_t, clone_idxs) / sizeof(void*)] = create_gv("jl_clone_idxs", true);
}
auto tables_arr = ConstantArray::get(ArrayType::get(T_psize, tables.size()), tables);
auto tables_gv = new GlobalVariable(M, tables_arr->getType(), false,
GlobalValue::ExternalLinkage, tables_arr, "jl_shard_tables");
tables_gv->setVisibility(GlobalValue::HiddenVisibility);
return tables_gv;
}

static GlobalVariable *emit_ptls_table(Module &M, Type *T_size, Type *T_psize) {
std::array<Constant *, 3> ptls_table{
new GlobalVariable(M, T_size, false, GlobalValue::ExternalLinkage, Constant::getNullValue(T_size), "jl_pgcstack_func_slot"),
new GlobalVariable(M, T_size, false, GlobalValue::ExternalLinkage, Constant::getNullValue(T_size), "jl_pgcstack_key_slot"),
new GlobalVariable(M, T_size, false, GlobalValue::ExternalLinkage, Constant::getNullValue(T_size), "jl_tls_offset"),
};
for (auto &gv : ptls_table)
cast<GlobalVariable>(gv)->setVisibility(GlobalValue::HiddenVisibility);
auto ptls_table_arr = ConstantArray::get(ArrayType::get(T_psize, ptls_table.size()), ptls_table);
auto ptls_table_gv = new GlobalVariable(M, ptls_table_arr->getType(), false,
GlobalValue::ExternalLinkage, ptls_table_arr, "jl_ptls_table");
ptls_table_gv->setVisibility(GlobalValue::HiddenVisibility);
return ptls_table_gv;
}

static GlobalVariable *emit_image_header(Module &M, unsigned threads, unsigned nfvars, unsigned ngvars) {
constexpr uint32_t version = 1;
std::array<uint32_t, 4> header{
version,
threads,
nfvars,
ngvars,
};
auto header_arr = ConstantDataArray::get(M.getContext(), header);
auto header_gv = new GlobalVariable(M, header_arr->getType(), false,
GlobalValue::InternalLinkage, header_arr, "jl_image_header");
return header_gv;
}

// takes the running content that has collected in the shadow module and dump it to disk
// this builds the object file portion of the sysimage files for fast startup
extern "C" JL_DLLEXPORT
Expand Down Expand Up @@ -588,6 +646,10 @@ void jl_dump_native_impl(void *native_code,

start = jl_hrtime();

unsigned threads = 1;
unsigned nfvars = 0;
unsigned ngvars = 0;

// add metadata information
if (imaging_mode) {
multiversioning_preannotate(*dataM);
Expand All @@ -601,8 +663,27 @@ void jl_dump_native_impl(void *native_code,
}
}
}
emit_offset_table(*dataM, data->jl_sysimg_gvars, "jl_sysimg_gvars", T_psize);
emit_offset_table(*dataM, data->jl_sysimg_fvars, "jl_sysimg_fvars", T_psize);
nfvars = data->jl_sysimg_fvars.size();
ngvars = data->jl_sysimg_gvars.size();
emit_offset_table(*dataM, data->jl_sysimg_gvars, "jl_gvars", T_psize);
emit_offset_table(*dataM, data->jl_sysimg_fvars, "jl_fvars", T_psize);
std::vector<uint32_t> idxs;
idxs.resize(data->jl_sysimg_gvars.size());
std::iota(idxs.begin(), idxs.end(), 0);
auto gidxs = ConstantDataArray::get(Context, idxs);
auto gidxs_var = new GlobalVariable(*dataM, gidxs->getType(), true,
GlobalVariable::ExternalLinkage,
gidxs, "jl_gvar_idxs");
gidxs_var->setVisibility(GlobalValue::HiddenVisibility);
idxs.clear();
idxs.resize(data->jl_sysimg_fvars.size());
std::iota(idxs.begin(), idxs.end(), 0);
auto fidxs = ConstantDataArray::get(Context, idxs);
auto fidxs_var = new GlobalVariable(*dataM, fidxs->getType(), true,
GlobalVariable::ExternalLinkage,
fidxs, "jl_fvar_idxs");
fidxs_var->setVisibility(GlobalValue::HiddenVisibility);
dataM->addModuleFlag(Module::Error, "julia.mv.suffix", MDString::get(Context, "_0"));

// reflect the address of the jl_RTLD_DEFAULT_handle variable
// back to the caller, so that we can check for consistency issues
Expand Down Expand Up @@ -789,10 +870,23 @@ void jl_dump_native_impl(void *native_code,
data.insert(data.end(), specdata.begin(), specdata.end());
}
auto value = ConstantDataArray::get(Context, data);
addComdat(new GlobalVariable(*sysimageM, value->getType(), true,
GlobalVariable::ExternalLinkage,
value, "jl_dispatch_target_ids"));

auto target_ids = new GlobalVariable(*sysimageM, value->getType(), true,
GlobalVariable::InternalLinkage,
value, "jl_dispatch_target_ids");
auto shards = emit_shard_table(*sysimageM, T_size, T_psize, threads);
auto ptls = emit_ptls_table(*sysimageM, T_size, T_psize);
auto header = emit_image_header(*sysimageM, threads, nfvars, ngvars);
auto AT = ArrayType::get(T_psize, 4);
auto pointers = new GlobalVariable(*sysimageM, AT, false,
GlobalVariable::ExternalLinkage,
ConstantArray::get(AT, {
ConstantExpr::getBitCast(header, T_psize),
ConstantExpr::getBitCast(shards, T_psize),
ConstantExpr::getBitCast(ptls, T_psize),
ConstantExpr::getBitCast(target_ids, T_psize)
}),
"jl_image_pointers");
addComdat(pointers);
if (s) {
write_int32(s, data.size());
ios_write(s, (const char *)data.data(), data.size());
Expand Down
68 changes: 35 additions & 33 deletions src/llvm-multiversioning.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -516,8 +516,8 @@ static inline std::vector<T*> consume_gv(Module &M, const char *name, bool allow
CloneCtx::CloneCtx(Module &M, bool allow_bad_fvars)
: tbaa_const(tbaa_make_child_with_context(M.getContext(), "jtbaa_const", nullptr, true).first),
specs(jl_get_llvm_clone_targets()),
fvars(consume_gv<Function>(M, "jl_sysimg_fvars", allow_bad_fvars)),
gvars(consume_gv<Constant>(M, "jl_sysimg_gvars", false)),
fvars(consume_gv<Function>(M, "jl_fvars", allow_bad_fvars)),
gvars(consume_gv<Constant>(M, "jl_gvars", false)),
M(M),
allow_bad_fvars(allow_bad_fvars)
{
Expand Down Expand Up @@ -547,7 +547,7 @@ CloneCtx::CloneCtx(Module &M, bool allow_bad_fvars)
for (uint32_t i = 0; i < nfvars; i++)
func_ids[fvars[i]] = i + 1;
for (auto &F: M) {
if (F.empty())
if (F.empty() && !F.hasFnAttribute("julia.mv.clones"))
continue;
orig_funcs.push_back(&F);
}
Expand Down Expand Up @@ -898,19 +898,6 @@ void CloneCtx::fix_inst_uses()
}
}

template<typename T>
static inline T *add_comdat(T *G)
{
#if defined(_OS_WINDOWS_)
// add __declspec(dllexport) to everything marked for export
if (G->getLinkage() == GlobalValue::ExternalLinkage)
G->setDLLStorageClass(GlobalValue::DLLExportStorageClass);
else
G->setDLLStorageClass(GlobalValue::DefaultStorageClass);
#endif
return G;
}

static Constant *get_ptrdiff32(Constant *ptr, Constant *base)
{
if (ptr->getType()->isPointerTy())
Expand All @@ -920,19 +907,22 @@ static Constant *get_ptrdiff32(Constant *ptr, Constant *base)
}

template<typename T>
static Constant *emit_offset_table(Module &M, const std::vector<T*> &vars, StringRef name)
static Constant *emit_offset_table(Module &M, const std::vector<T*> &vars, StringRef name, StringRef suffix)
{
auto T_int32 = Type::getInt32Ty(M.getContext());
auto T_size = getSizeTy(M.getContext());
uint32_t nvars = vars.size();
Constant *base = nullptr;
if (nvars > 0) {
base = ConstantExpr::getBitCast(vars[0], T_size->getPointerTo());
add_comdat(GlobalAlias::create(T_size, 0, GlobalVariable::ExternalLinkage,
name + "_base",
base, &M));
auto ga = GlobalAlias::create(T_size, 0, GlobalVariable::ExternalLinkage,
name + "_base" + suffix,
base, &M);
ga->setVisibility(GlobalValue::HiddenVisibility);
} else {
base = add_comdat(new GlobalVariable(M, T_size, true, GlobalValue::ExternalLinkage, Constant::getNullValue(T_size), name + "_base"));
auto gv = new GlobalVariable(M, T_size, true, GlobalValue::ExternalLinkage, Constant::getNullValue(T_size), name + "_base" + suffix);
gv->setVisibility(GlobalValue::HiddenVisibility);
base = gv;
}
auto vbase = ConstantExpr::getPtrToInt(base, T_size);
std::vector<Constant*> offsets(nvars + 1);
Expand All @@ -943,10 +933,11 @@ static Constant *emit_offset_table(Module &M, const std::vector<T*> &vars, Strin
offsets[i + 1] = get_ptrdiff32(vars[i], vbase);
}
ArrayType *vars_type = ArrayType::get(T_int32, nvars + 1);
add_comdat(new GlobalVariable(M, vars_type, true,
auto gv = new GlobalVariable(M, vars_type, true,
GlobalVariable::ExternalLinkage,
ConstantArray::get(vars_type, offsets),
name + "_offsets"));
name + "_offsets" + suffix);
gv->setVisibility(GlobalValue::HiddenVisibility);
return vbase;
}

Expand All @@ -958,9 +949,17 @@ void CloneCtx::emit_metadata()
return;
}

StringRef suffix;
if (auto suffix_md = M.getModuleFlag("julia.mv.suffix")) {
suffix = cast<MDString>(suffix_md)->getString();
}

// Store back the information about exported functions.
auto fbase = emit_offset_table(M, fvars, "jl_sysimg_fvars");
auto gbase = emit_offset_table(M, gvars, "jl_sysimg_gvars");
auto fbase = emit_offset_table(M, fvars, "jl_fvar", suffix);
auto gbase = emit_offset_table(M, gvars, "jl_gvar", suffix);

M.getGlobalVariable("jl_fvar_idxs")->setName("jl_fvar_idxs" + suffix);
M.getGlobalVariable("jl_gvar_idxs")->setName("jl_gvar_idxs" + suffix);

uint32_t ntargets = specs.size();

Expand Down Expand Up @@ -996,9 +995,10 @@ void CloneCtx::emit_metadata()
}
values[0] = ConstantInt::get(T_int32, values.size() / 2);
ArrayType *vars_type = ArrayType::get(T_int32, values.size());
add_comdat(new GlobalVariable(M, vars_type, true, GlobalVariable::ExternalLinkage,
auto gv = new GlobalVariable(M, vars_type, true, GlobalVariable::ExternalLinkage,
ConstantArray::get(vars_type, values),
"jl_dispatch_reloc_slots"));
"jl_clone_slots" + suffix);
gv->setVisibility(GlobalValue::HiddenVisibility);
}

// Generate `jl_dispatch_fvars_idxs` and `jl_dispatch_fvars_offsets`
Expand Down Expand Up @@ -1046,14 +1046,16 @@ void CloneCtx::emit_metadata()
idxs[len_idx] = count;
}
auto idxval = ConstantDataArray::get(M.getContext(), idxs);
add_comdat(new GlobalVariable(M, idxval->getType(), true,
auto gv1 = new GlobalVariable(M, idxval->getType(), true,
GlobalVariable::ExternalLinkage,
idxval, "jl_dispatch_fvars_idxs"));
idxval, "jl_clone_idxs" + suffix);
gv1->setVisibility(GlobalValue::HiddenVisibility);
ArrayType *offsets_type = ArrayType::get(Type::getInt32Ty(M.getContext()), offsets.size());
add_comdat(new GlobalVariable(M, offsets_type, true,
auto gv2 = new GlobalVariable(M, offsets_type, true,
GlobalVariable::ExternalLinkage,
ConstantArray::get(offsets_type, offsets),
"jl_dispatch_fvars_offsets"));
"jl_clone_offsets" + suffix);
gv2->setVisibility(GlobalValue::HiddenVisibility);
}
}

Expand All @@ -1070,8 +1072,8 @@ static bool runMultiVersioning(Module &M, bool allow_bad_fvars)
if (M.getName() == "sysimage")
return false;

GlobalVariable *fvars = M.getGlobalVariable("jl_sysimg_fvars");
GlobalVariable *gvars = M.getGlobalVariable("jl_sysimg_gvars");
GlobalVariable *fvars = M.getGlobalVariable("jl_fvars");
GlobalVariable *gvars = M.getGlobalVariable("jl_gvars");
if (allow_bad_fvars && (!fvars || !fvars->hasInitializer() || !isa<ConstantArray>(fvars->getInitializer()) ||
!gvars || !gvars->hasInitializer() || !isa<ConstantArray>(gvars->getInitializer())))
return false;
Expand Down
19 changes: 2 additions & 17 deletions src/llvm-ptls.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,26 +140,11 @@ GlobalVariable *LowerPTLS::create_aliased_global(Type *T, StringRef name) const
// the address is visible externally but LLVM can still assume that the
// address of this variable doesn't need dynamic relocation
// (can be accessed with a single PC-rel load).
auto GV = new GlobalVariable(*M, T, false, GlobalVariable::InternalLinkage,
Constant::getNullValue(T), name + ".real");
add_comdat(GlobalAlias::create(T, 0, GlobalVariable::ExternalLinkage,
name, GV, M));
auto GV = new GlobalVariable(*M, T, false, GlobalVariable::ExternalLinkage,
nullptr, name);
return GV;
}

template<typename T>
inline T *LowerPTLS::add_comdat(T *G) const
{
#if defined(_OS_WINDOWS_)
// add __declspec(dllexport) to everything marked for export
if (G->getLinkage() == GlobalValue::ExternalLinkage)
G->setDLLStorageClass(GlobalValue::DLLExportStorageClass);
else
G->setDLLStorageClass(GlobalValue::DefaultStorageClass);
#endif
return G;
}

void LowerPTLS::fix_pgcstack_use(CallInst *pgcstack, Function *pgcstack_getter, bool or_new, bool *CFGModified)
{
if (pgcstack->use_empty()) {
Expand Down
Loading

0 comments on commit 6ab1862

Please sign in to comment.