stall-detector: Try hard not to crash while collecting backtrace

include/seastar/core/reactor.hh

@@ -655,6 +655,7 @@ public:
         /// resets the supression state.
         static void set_stall_detector_report_function(std::function<void ()> report);
         static std::function<void ()> get_stall_detector_report_function();
+        static void set_stall_detector_crash_collecting_backtrace();
     };
 };
 

src/core/reactor.cc

@@ -32,6 +32,7 @@ module;
 #include <fstream>
 #include <regex>
 #include <thread>
+#include <setjmp.h>
 
 #include <spawn.h>
 #include <sys/syscall.h>
@@ -842,7 +843,23 @@ class backtrace_buffer {
     }
 };
 
+static thread_local jmp_buf stall_detector_env;
+static thread_local bool in_stall_detector = false;
+static thread_local bool crash_collecting_backtrace = false;
+
+inline void maybe_crash_for_test() noexcept {
+    if (crash_collecting_backtrace) [[unlikely]] {
+        *(volatile int *)nullptr = 0;
+    }
+}
+
 static void print_with_backtrace(backtrace_buffer& buf, bool oneline) noexcept {
+    if (sigsetjmp(stall_detector_env, 0)) {
+        buf.append(" ¯\\_(ツ)_/¯\n");
+        goto out;
+    }
+    in_stall_detector = true;
+
     if (local_engine) {
         buf.append(" on shard ");
         buf.append_decimal(this_shard_id());
@@ -853,12 +870,17 @@ static void print_with_backtrace(backtrace_buffer& buf, bool oneline) noexcept {
 
   if (!oneline) {
     buf.append(".\nBacktrace:\n");
+    maybe_crash_for_test();
     buf.append_backtrace();
   } else {
     buf.append(". Backtrace:");
+    maybe_crash_for_test();
     buf.append_backtrace_oneline();
     buf.append("\n");
   }
+
+out:
+    in_stall_detector = false;
     buf.flush();
 }
 
@@ -1502,6 +1524,10 @@ reactor::test::set_stall_detector_report_function(std::function<void ()> report)
     r._cpu_stall_detector->reset_suppression_state(reactor::now());
 }
 
+void reactor::test::set_stall_detector_crash_collecting_backtrace() {
+    crash_collecting_backtrace = true;
+}
+
 std::function<void ()>
 reactor::test::get_stall_detector_report_function() {
     return engine()._cpu_stall_detector->get_config().report;
@@ -3930,6 +3956,9 @@ void install_oneshot_signal_handler() {
 
     struct sigaction sa;
     sa.sa_sigaction = [](int sig, siginfo_t *info, void *p) {
+        if (sig == SIGSEGV && in_stall_detector) {
+            siglongjmp(stall_detector_env, 1);
+        }
         std::lock_guard<util::spinlock> g(lock);
         if (!handled) {
             handled = true;

tests/unit/stall_detector_test.cc

@@ -175,6 +175,11 @@ SEASTAR_THREAD_TEST_CASE(spin_in_kernel) {
     test_spin_with_body("kernel", [] { mmap_populate(128 * 1024); });
 }
 
+SEASTAR_THREAD_TEST_CASE(crash_collecting_backtrace) {
+    reactor::test::set_stall_detector_crash_collecting_backtrace();
+    engine().update_blocked_reactor_notify_ms(100ms);
+    spin(500ms);
+}
 
 #else