From 2939272af2ef3fe9d8921f7ed0a6500e31a550c9 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 7 Jan 2022 12:21:35 -0500 Subject: [PATCH 1/2] macOS: workaround a dyld/libunwind deadlock issue since 12.1 Apple reintroduced the old bug that we previously worked around in fad04d39d592d8e0fcbfba439e8157f582bbc850 with a similar patch to this. This is needed anywhere that we may attempt to stop threads. Fixes #43578 --- src/Makefile | 1 + src/mach_dyld_atfork.tbd | 25 +++++++++++++++++++++++++ src/signals-mach.c | 31 +++++++++++++++++++++++++++---- 3 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 src/mach_dyld_atfork.tbd diff --git a/src/Makefile b/src/Makefile index cda08995f5538..0f6f9abd5ee95 100644 --- a/src/Makefile +++ b/src/Makefile @@ -145,6 +145,7 @@ endif CLANG_LDFLAGS := $(LLVM_LDFLAGS) ifeq ($(OS), Darwin) CLANG_LDFLAGS += -Wl,-undefined,dynamic_lookup +OSLIBS += $(SRCDIR)/mach_dyld_atfork.tbd endif COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir) diff --git a/src/mach_dyld_atfork.tbd b/src/mach_dyld_atfork.tbd new file mode 100644 index 0000000000000..9a5d18099dbcf --- /dev/null +++ b/src/mach_dyld_atfork.tbd @@ -0,0 +1,25 @@ +--- !tapi-tbd +# copied from XCode's libSystem.tbd (current-version: 1311) +# to provide weak-linkage info for new symbols on old systems +tbd-version: 4 +targets: [ x86_64-macos, x86_64-maccatalyst, arm64-macos, arm64-maccatalyst, + arm64e-macos, arm64e-maccatalyst ] +uuids: + - target: x86_64-macos + value: AFE6C76A-B47A-35F5-91D0-4E9FC439E90D + - target: x86_64-maccatalyst + value: AFE6C76A-B47A-35F5-91D0-4E9FC439E90D + - target: arm64-macos + value: 2EA09BDB-811B-33AA-BB58-4B53AA2DB522 + - target: arm64-maccatalyst + value: 2EA09BDB-811B-33AA-BB58-4B53AA2DB522 + - target: arm64e-macos + value: 09AB3723-C26D-3762-93BA-98E9C38B89C1 + - target: arm64e-maccatalyst + value: 09AB3723-C26D-3762-93BA-98E9C38B89C1 +install-name: '/usr/lib/libSystem.B.dylib' +exports: + - targets: [ arm64-macos, arm64e-macos, x86_64-macos, x86_64-maccatalyst, + arm64-maccatalyst, arm64e-maccatalyst ] + symbols: [ __dyld_atfork_parent, __dyld_atfork_prepare ] +... diff --git a/src/signals-mach.c b/src/signals-mach.c index 57dcc969068e8..364c3dc64d667 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -31,6 +31,11 @@ extern void *_keymgr_get_and_lock_processwide_ptr(unsigned int key); extern int _keymgr_get_and_lock_processwide_ptr_2(unsigned int key, void **result); extern int _keymgr_set_lockmode_processwide_ptr(unsigned int key, unsigned int mode); +// private dyld3/dyld4 stuff +extern void _dyld_atfork_prepare(void) __attribute__((weak_import)); +extern void _dyld_atfork_parent(void) __attribute__((weak_import)); +//extern void _dyld_fork_child(void) __attribute__((weak_import)); + static void attach_exception_port(thread_port_t thread, int segv_only); // low 16 bits are the thread id, the next 8 bits are the original gc_state @@ -521,6 +526,28 @@ static kern_return_t profiler_segv_handler } #endif +static int jl_lock_profile_mach(void) +{ + jl_lock_profile(); + void *unused = NULL; + int keymgr_locked = _keymgr_get_and_lock_processwide_ptr_2(KEYMGR_GCC3_DW2_OBJ_LIST, &unused) == 0; + if (_dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) + _dyld_atfork_prepare(); + return keymgr_locked; +} + +static void jl_unlock_profile_mach(int keymgr_locked) +{ + if (_dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) + _dyld_atfork_parent(); + if (keymgr_locked) + _keymgr_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST); + jl_unlock_profile(); +} + +#define jl_lock_profile() int keymgr_locked = jl_lock_profile_mach() +#define jl_unlock_profile() jl_unlock_profile_mach(keymgr_locked) + void *mach_profile_listener(void *arg) { (void)arg; @@ -538,8 +565,6 @@ void *mach_profile_listener(void *arg) // sample each thread, round-robin style in reverse order // (so that thread zero gets notified last) jl_lock_profile(); - void *unused = NULL; - int keymgr_locked = _keymgr_get_and_lock_processwide_ptr_2(KEYMGR_GCC3_DW2_OBJ_LIST, &unused) == 0; jl_shuffle_int_array_inplace(profile_round_robin_thread_order, jl_n_threads, &profile_cong_rng_seed); for (int idx = jl_n_threads; idx-- > 0; ) { // Stop the threads in the random round-robin order. @@ -609,8 +634,6 @@ void *mach_profile_listener(void *arg) // We're done! Resume the thread. jl_thread_resume(i, 0); } - if (keymgr_locked) - _keymgr_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST); jl_unlock_profile(); if (running) { // Reset the alarm From 267b124f1f1f1558a7619a527e11a786a99f01d4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 12 Jan 2022 00:20:32 -0500 Subject: [PATCH 2/2] macOS: extend the workaround to cover the dyld/exc_server deadlock issue, since 12.1 Later, we should probably switch to using mach_exc_server generated from `mig mach_exc.defs`. --- src/signals-mach.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/signals-mach.c b/src/signals-mach.c index 364c3dc64d667..4ddfad4e10600 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -526,27 +526,30 @@ static kern_return_t profiler_segv_handler } #endif -static int jl_lock_profile_mach(void) +// WARNING: we are unable to handle sigsegv while the dlsymlock is held +static int jl_lock_profile_mach(int dlsymlock) { jl_lock_profile(); + // workaround for old keymgr bugs void *unused = NULL; int keymgr_locked = _keymgr_get_and_lock_processwide_ptr_2(KEYMGR_GCC3_DW2_OBJ_LIST, &unused) == 0; - if (_dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) + // workaround for new dlsym4 bugs (API and bugs introduced in macOS 12.1) + if (dlsymlock && _dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) _dyld_atfork_prepare(); return keymgr_locked; } -static void jl_unlock_profile_mach(int keymgr_locked) +static void jl_unlock_profile_mach(int dlsymlock, int keymgr_locked) { - if (_dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) - _dyld_atfork_parent(); + if (dlsymlock && _dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) \ + _dyld_atfork_parent(); \ if (keymgr_locked) _keymgr_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST); jl_unlock_profile(); } -#define jl_lock_profile() int keymgr_locked = jl_lock_profile_mach() -#define jl_unlock_profile() jl_unlock_profile_mach(keymgr_locked) +#define jl_lock_profile() int keymgr_locked = jl_lock_profile_mach(1) +#define jl_unlock_profile() jl_unlock_profile_mach(1, keymgr_locked) void *mach_profile_listener(void *arg) { @@ -564,7 +567,7 @@ void *mach_profile_listener(void *arg) HANDLE_MACH_ERROR("mach_msg", ret); // sample each thread, round-robin style in reverse order // (so that thread zero gets notified last) - jl_lock_profile(); + int keymgr_locked = jl_lock_profile_mach(0); jl_shuffle_int_array_inplace(profile_round_robin_thread_order, jl_n_threads, &profile_cong_rng_seed); for (int idx = jl_n_threads; idx-- > 0; ) { // Stop the threads in the random round-robin order. @@ -575,9 +578,13 @@ void *mach_profile_listener(void *arg) break; } + if (_dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) + _dyld_atfork_prepare(); // briefly acquire the dlsym lock host_thread_state_t state; jl_thread_suspend_and_get_state2(i, &state); unw_context_t *uc = (unw_context_t*)&state; + if (_dyld_atfork_prepare != NULL && _dyld_atfork_parent != NULL) + _dyld_atfork_parent(); // quickly release the dlsym lock if (running) { #ifdef LLVMLIBUNWIND @@ -634,7 +641,7 @@ void *mach_profile_listener(void *arg) // We're done! Resume the thread. jl_thread_resume(i, 0); } - jl_unlock_profile(); + jl_unlock_profile_mach(0, keymgr_locked); if (running) { // Reset the alarm kern_return_t ret = clock_alarm(clk, TIME_RELATIVE, timerprof, profile_port);