diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 7ef3b12cd08da..27cc2659261a1 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -493,6 +493,7 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.instrument_coverage, config.instrument_gcov, + config.instrument_mcount, pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), config.debug_info_for_profiling, llvm_selfprofiler, @@ -573,6 +574,7 @@ pub(crate) unsafe fn optimize( llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); } + let mut early_passes = Vec::new(); let mut extra_passes = Vec::new(); let mut have_name_anon_globals_pass = false; @@ -609,6 +611,12 @@ pub(crate) unsafe fn optimize( llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); } + // EntryExitInstrumentation needs to be added by the frontend as of LLVM 13 + if config.instrument_mcount && llvm_util::get_version() >= (13, 0, 0) { + early_passes.push(llvm::LLVMRustCreateEntryExitInstrumenterPass(false)); + extra_passes.push(llvm::LLVMRustCreateEntryExitInstrumenterPass(true)); + } + add_sanitizer_passes(config, &mut extra_passes); // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need @@ -623,6 +631,11 @@ pub(crate) unsafe fn optimize( || cgcx.lto == Lto::ThinLocal || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { + llvm::LLVMRustAddEarlyExtensionPasses( + b, + early_passes.as_ptr(), + early_passes.len() as size_t, + ); llvm::LLVMRustAddLastExtensionPasses( b, extra_passes.as_ptr(), @@ -640,7 +653,7 @@ pub(crate) unsafe fn optimize( } else { // If we don't use the standard pipeline, directly populate the MPM // with the extra passes. - for pass in extra_passes { + for pass in early_passes.into_iter().chain(extra_passes.into_iter()) { llvm::LLVMRustAddPass(mpm, pass); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 7f533b0552a5d..b50e2c82621f4 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -2253,7 +2253,13 @@ extern "C" { ) -> &'static mut Pass; pub fn LLVMRustCreateThreadSanitizerPass() -> &'static mut Pass; pub fn LLVMRustCreateHWAddressSanitizerPass(Recover: bool) -> &'static mut Pass; + pub fn LLVMRustCreateEntryExitInstrumenterPass(PostInlining: bool) -> &'static mut Pass; pub fn LLVMRustAddPass(PM: &PassManager<'_>, Pass: &'static mut Pass); + pub fn LLVMRustAddEarlyExtensionPasses( + PMB: &PassManagerBuilder, + Passes: *const &'static mut Pass, + NumPasses: size_t, + ); pub fn LLVMRustAddLastExtensionPasses( PMB: &PassManagerBuilder, Passes: *const &'static mut Pass, @@ -2342,6 +2348,7 @@ extern "C" { PGOUsePath: *const c_char, InstrumentCoverage: bool, InstrumentGCOV: bool, + InstrumentMcount: bool, PGOSampleUsePath: *const c_char, DebugInfoForProfiling: bool, llvm_selfprofiler: *mut c_void, diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs index d5d21775f0abc..60b86823d9a11 100644 --- a/compiler/rustc_codegen_ssa/src/back/write.rs +++ b/compiler/rustc_codegen_ssa/src/back/write.rs @@ -87,6 +87,7 @@ pub struct ModuleConfig { pub debug_info_for_profiling: bool, pub instrument_coverage: bool, pub instrument_gcov: bool, + pub instrument_mcount: bool, pub sanitizer: SanitizerSet, pub sanitizer_recover: SanitizerSet, @@ -189,6 +190,7 @@ impl ModuleConfig { sess.opts.debugging_opts.profile && !is_compiler_builtins, false ), + instrument_mcount: if_regular!(sess.instrument_mcount(), false), sanitizer: if_regular!(sess.opts.debugging_opts.sanitizer, SanitizerSet::empty()), sanitizer_recover: if_regular!( diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp index 71f21dc666686..394c85f102880 100644 --- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp @@ -32,6 +32,7 @@ #include "llvm/Transforms/IPO/AlwaysInliner.h" #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/Transforms/Utils/AddDiscriminators.h" +#include "llvm/Transforms/Utils/EntryExitInstrumenter.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/LTO/LTO.h" #include "llvm-c/Transforms/PassManagerBuilder.h" @@ -147,6 +148,18 @@ extern "C" LLVMPassRef LLVMRustCreateHWAddressSanitizerPass(bool Recover) { return wrap(createHWAddressSanitizerLegacyPassPass(CompileKernel, Recover)); } +extern "C" LLVMPassRef LLVMRustCreateEntryExitInstrumenterPass(bool PostInlining) { +#if LLVM_VERSION_LT(15, 0) + if (PostInlining) { + return wrap(createPostInlineEntryExitInstrumenterPass()); + } else { + return wrap(createEntryExitInstrumenterPass()); + } +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + extern "C" LLVMRustPassKind LLVMRustPassKind(LLVMPassRef RustPass) { assert(RustPass); Pass *Pass = unwrap(RustPass); @@ -168,6 +181,25 @@ void LLVMRustPassManagerBuilderPopulateThinLTOPassManager( unwrap(PMBR)->populateThinLTOPassManager(*unwrap(PMR)); } +extern "C" +void LLVMRustAddEarlyExtensionPasses( + LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { +#if LLVM_VERSION_LT(15, 0) + auto AddExtensionPasses = [Passes, NumPasses]( + const PassManagerBuilder &Builder, PassManagerBase &PM) { + for (size_t I = 0; I < NumPasses; I++) { + PM.add(unwrap(Passes[I])); + } + }; + unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EarlyAsPossible, + AddExtensionPasses); + unwrap(PMBR)->addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0, + AddExtensionPasses); +#else + report_fatal_error("Legacy PM not supported with LLVM 15"); +#endif +} + extern "C" void LLVMRustAddLastExtensionPasses( LLVMPassManagerBuilderRef PMBR, LLVMPassRef *Passes, size_t NumPasses) { @@ -749,7 +781,7 @@ LLVMRustOptimizeWithNewPassManager( bool DisableSimplifyLibCalls, bool EmitLifetimeMarkers, LLVMRustSanitizerOptions *SanitizerOptions, const char *PGOGenPath, const char *PGOUsePath, - bool InstrumentCoverage, bool InstrumentGCOV, + bool InstrumentCoverage, bool InstrumentGCOV, bool InstrumentMcount, const char *PGOSampleUsePath, bool DebugInfoForProfiling, void* LlvmSelfProfiler, LLVMRustSelfProfileBeforePassCallback BeforePassCallback, @@ -856,6 +888,26 @@ LLVMRustOptimizeWithNewPassManager( ); } +// EntryExitInstrumentation needs to be added by the frontend as of LLVM 13 +#if LLVM_VERSION_GE(13, 0) + if (InstrumentMcount) { + PipelineStartEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(createModuleToFunctionPassAdaptor( + EntryExitInstrumenterPass(/*PostInlining=*/false) + )); + } + ); + OptimizerLastEPCallbacks.push_back( + [](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(createModuleToFunctionPassAdaptor( + EntryExitInstrumenterPass(/*PostInlining=*/true) + )); + } + ); + } +#endif + if (SanitizerOptions) { if (SanitizerOptions->SanitizeMemory) { MemorySanitizerOptions Options( diff --git a/src/test/codegen/instrument-mcount-legacy.rs b/src/test/codegen/instrument-mcount-legacy.rs new file mode 100644 index 0000000000000..5bc94e5b52daf --- /dev/null +++ b/src/test/codegen/instrument-mcount-legacy.rs @@ -0,0 +1,9 @@ +// ignore-llvm-version: 13.0.0 - 99.99.99 +// revisions: opt noopt +// compile-flags: -Z instrument-mcount +// [opt]compile-flags: -O + +#![crate_type = "lib"] + +// CHECK: attributes #{{.*}} "frame-pointer"="all" "instrument-function-entry-inlined"="{{.*}}mcount{{.*}}" +pub fn foo() {} diff --git a/src/test/codegen/instrument-mcount.rs b/src/test/codegen/instrument-mcount.rs index b26076e7a7bfa..e218c9eb849fe 100644 --- a/src/test/codegen/instrument-mcount.rs +++ b/src/test/codegen/instrument-mcount.rs @@ -1,7 +1,9 @@ -// +// min-llvm-version: 13.0.0 +// revisions: opt noopt // compile-flags: -Z instrument-mcount +// [opt]compile-flags: -O #![crate_type = "lib"] -// CHECK: attributes #{{.*}} "frame-pointer"="all" "instrument-function-entry-inlined"="{{.*}}mcount{{.*}}" +// CHECK: call void @{{.*}}mcount pub fn foo() {}