Source file and line numbers in stacktrace.

* File & line will also show in cf_thread traces.

Co-authored-by: Andrew Gooding <[email protected]>

.gitmodules

@@ -27,3 +27,7 @@
 	path = modules/s2geometry
 	url = https://github.com/aerospike/s2geometry.git
 	ignore = dirty
+[submodule "modules/libbacktrace"]
+	path = modules/libbacktrace
+	url = https://github.com/aerospike/libbacktrace.git
+	ignore = dirty

Makefile

@@ -49,12 +49,13 @@ lib: aslibs
 	$(MAKE) -C as $@ STATIC_LIB=1 OS=$(OS)
 
 .PHONY: aslibs
-aslibs: targetdirs version $(JANSSON)/Makefile $(JEMALLOC)/Makefile $(LUAJIT)/src/luaconf.h s2lib
+aslibs: targetdirs version $(JANSSON)/Makefile $(JEMALLOC)/Makefile $(LIBBACKTRACE)/Makefile $(LUAJIT)/src/luaconf.h s2lib
+	$(MAKE) -C $(JANSSON)
+	$(MAKE) -C $(JEMALLOC)
+	$(MAKE) -C $(LIBBACKTRACE)
 ifeq ($(USE_LUAJIT),1)
 	$(MAKE) -C $(LUAJIT) Q= TARGET_SONAME=libluajit.so CCDEBUG=-g
 endif
-	$(MAKE) -C $(JEMALLOC)
-	$(MAKE) -C $(JANSSON)
 ifeq ($(ARCH), aarch64)
 	$(MAKE) -C $(TSO)
 endif
@@ -133,6 +134,9 @@ cleanmodules:
 		$(MAKE) -C $(JEMALLOC) clean; \
 		$(MAKE) -C $(JEMALLOC) distclean; \
 	fi
+	if [ -e "$(LIBBACKTRACE)/Makefile" ]; then \
+		$(MAKE) -C $(LIBBACKTRACE) clean; \
+	fi
 	if [ -e "$(LUAJIT)/Makefile" ]; then \
 		$(MAKE) -C $(LUAJIT) clean; \
 	fi
@@ -158,6 +162,7 @@ cleangit:
 	cd $(COMMON); $(GIT_CLEAN)
 	cd $(JANSSON); $(GIT_CLEAN)
 	cd $(JEMALLOC); $(GIT_CLEAN)
+	cd $(LIBBACKTRACE); $(GIT_CLEAN)
 	cd $(LUAJIT); $(GIT_CLEAN)
 	cd $(MOD_LUA); $(GIT_CLEAN)
 	cd $(S2); $(GIT_CLEAN)
@@ -188,6 +193,9 @@ $(JEMALLOC)/configure:
 $(JEMALLOC)/Makefile: $(JEMALLOC)/configure
 	cd $(JEMALLOC) && ./configure $(JEM_CONFIG_OPT)
 
+$(LIBBACKTRACE)/Makefile: $(LIBBACKTRACE)/configure
+	cd $(LIBBACKTRACE) && ./configure $(LIBBACKTRACE_CONFIG_OPT)
+
 .PHONY: source
 source: src
 

as/src/Makefile

@@ -231,12 +231,12 @@ SOURCES += $(XDR_SOURCES:%=xdr/%)
 SERVER = $(BIN_DIR)/asd
 
 INCLUDES += $(INCLUDE_DIR:%=-I%)
+INCLUDES += -I$(ABSL)
 INCLUDES += -I$(CF)/include
 INCLUDES += -I$(COMMON)/target/$(PLATFORM)/include
 INCLUDES += -I$(JANSSON)/src
 INCLUDES += -I$(MOD_LUA)/target/$(PLATFORM)/include
 INCLUDES += -I$(S2)/src
-INCLUDES += -I$(ABSL)
 
 ifeq ($(USE_LUAJIT),1)
   INCLUDES += -I$(LUAJIT)/src
@@ -251,6 +251,7 @@ endif
 AS_LIBRARIES += $(LIBRARY_DIR)/libcf.a
 AS_LIBRARIES += $(COMMON)/target/$(PLATFORM)/lib/libaerospike-common.a
 AS_LIBRARIES += $(JEMALLOC)/lib/libjemalloc.a
+AS_LIBRARIES += $(LIBBACKTRACE)/.libs/libbacktrace.a
 AS_LIBRARIES += $(MOD_LUA)/target/$(PLATFORM)/lib/libmod_lua.a
 
 ifeq ($(DOPROFILE),1)

as/src/base/as.c

@@ -48,6 +48,7 @@
 #include "log.h"
 #include "os.h"
 #include "tls.h"
+#include "trace.h"
 
 #include "base/batch.h"
 #include "base/cfg.h"
@@ -259,6 +260,7 @@ as_run(int argc, char **argv)
 	// Initializations before config parsing.
 	cf_log_init(early_verbose);
 	cf_alloc_init();
+	cf_trace_init();
 	cf_thread_init();
 	as_signal_setup();
 	cf_fips_init();

as/src/base/signal.c

@@ -25,17 +25,21 @@
 //
 
 #include <inttypes.h>
+#include <link.h>
 #include <pthread.h>
 #include <signal.h>
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <ucontext.h>
 #include <unistd.h>
 
 #include "cf_thread.h"
 #include "log.h"
+#include "trace.h"
+
 #include "warnings.h"
 
 
@@ -122,7 +126,7 @@ log_abort(const char* signal)
 			*aerospike_build_ee_sha == '\0' ? "" : aerospike_build_ee_sha);
 }
 
-static inline const char*
+static const char*
 siginfo_code_str(const siginfo_t* info)
 {
 	// Strings placed corresponding to the numeric value of the constant. See:
@@ -176,7 +180,7 @@ siginfo_code_str(const siginfo_t* info)
 	return "(unimpl)";
 }
 
-static inline void
+static void
 log_siginfo(const siginfo_t* info)
 {
 	char buf[512];
@@ -213,6 +217,66 @@ log_siginfo(const siginfo_t* info)
 	cf_warning(AS_AS, "%s", buf);
 }
 
+// Unlike cf_log_stack_trace(), includes registers and addresses.
+static void
+log_stack_trace(void* ctx)
+{
+	ucontext_t* uc = (ucontext_t*)ctx;
+	mcontext_t* mc = (mcontext_t*)&uc->uc_mcontext;
+	char reg_str[1000];
+#if defined __x86_64__
+	uint64_t* gregs = (uint64_t*)&mc->gregs[0];
+
+	sprintf(reg_str,
+		"rax %016lx rbx %016lx rcx %016lx rdx %016lx rsi %016lx rdi %016lx "
+		"rbp %016lx rsp %016lx r8 %016lx r9 %016lx r10 %016lx r11 %016lx "
+		"r12 %016lx r13 %016lx r14 %016lx r15 %016lx rip %016lx",
+		gregs[REG_RAX], gregs[REG_RBX], gregs[REG_RCX], gregs[REG_RDX],
+		gregs[REG_RSI], gregs[REG_RDI], gregs[REG_RBP], gregs[REG_RSP],
+		gregs[REG_R8], gregs[REG_R9], gregs[REG_R10], gregs[REG_R11],
+		gregs[REG_R12], gregs[REG_R13], gregs[REG_R14], gregs[REG_R15],
+		gregs[REG_RIP]);
+#elif defined __aarch64__
+	uint64_t fault = (uint64_t)mc->fault_address;
+	uint64_t* regs = (uint64_t*)&mc->regs[0];
+	uint64_t sp = (uint64_t)mc->sp;
+	uint64_t pc = (uint64_t)mc->pc;
+
+	char* p = reg_str;
+
+	p += sprintf(p, "fault %016lx ", fault);
+
+	for (uint32_t i = 0; i < 31; i++) {
+		p += sprintf(p, "x%u %016lx ", i, regs[i]);
+	}
+
+	sprintf(p, "x31 %016lx pc %016lx", sp, pc);
+#endif
+
+	cf_warning(AS_AS, "stacktrace: registers: %s", reg_str);
+
+	void* bt[MAX_BACKTRACE_DEPTH];
+	char trace[MAX_BACKTRACE_DEPTH * 20];
+
+	int n_frames = cf_backtrace(bt, MAX_BACKTRACE_DEPTH);
+	int off = 0;
+
+	for (int i = 0; i < n_frames; i++) {
+		off += sprintf(trace + off, " 0x%lx", cf_strip_aslr(bt[i]));
+	}
+
+	cf_warning(AS_AS, "stacktrace: found %d frames:%s offset 0x%lx", n_frames,
+			trace, _r_debug.r_map->l_addr);
+
+	for (int i = 0; i < n_frames; i++) {
+		char sym_str[SYM_STR_MAX_SZ];
+
+		cf_addr_to_sym_str(sym_str, bt[i]);
+		cf_warning(AS_AS, "stacktrace: frame %d: %s [0x%lx]", i, sym_str,
+				cf_strip_aslr(bt[i]));
+	}
+}
+
 
 //==========================================================
 // Signal handlers.
@@ -224,7 +288,7 @@ as_sig_handle_abort(int sig_num, siginfo_t* info, void* ctx)
 {
 	log_abort("SIGABRT");
 	log_siginfo(info);
-	cf_log_stack_trace(ctx);
+	log_stack_trace(ctx);
 	reraise_signal(sig_num);
 }
 
@@ -233,7 +297,7 @@ as_sig_handle_bus(int sig_num, siginfo_t* info, void* ctx)
 {
 	log_abort("SIGBUS");
 	log_siginfo(info);
-	cf_log_stack_trace(ctx);
+	log_stack_trace(ctx);
 	reraise_signal(sig_num);
 }
 
@@ -243,7 +307,7 @@ as_sig_handle_fpe(int sig_num, siginfo_t* info, void* ctx)
 {
 	log_abort("SIGFPE");
 	log_siginfo(info);
-	cf_log_stack_trace(ctx);
+	log_stack_trace(ctx);
 	reraise_signal(sig_num);
 }
 
@@ -266,7 +330,7 @@ as_sig_handle_ill(int sig_num, siginfo_t* info, void* ctx)
 {
 	log_abort("SIGILL");
 	log_siginfo(info);
-	cf_log_stack_trace(ctx);
+	log_stack_trace(ctx);
 	reraise_signal(sig_num);
 }
 
@@ -295,7 +359,7 @@ as_sig_handle_quit(int sig_num, siginfo_t* info, void* ctx)
 {
 	log_abort("SIGQUIT");
 	log_siginfo(info);
-	cf_log_stack_trace(ctx);
+	log_stack_trace(ctx);
 	reraise_signal(sig_num);
 }
 
@@ -305,7 +369,7 @@ as_sig_handle_segv(int sig_num, siginfo_t* info, void* ctx)
 {
 	log_abort("SIGSEGV");
 	log_siginfo(info);
-	cf_log_stack_trace(ctx);
+	log_stack_trace(ctx);
 	reraise_signal(sig_num);
 }
 
@@ -342,7 +406,7 @@ as_sig_handle_usr1(int sig_num, siginfo_t* info, void* ctx)
 
 	log_abort("SIGUSR1");
 	log_siginfo(info);
-	cf_log_stack_trace(ctx);
+	log_stack_trace(ctx);
 	reraise_signal(SIGABRT);
 }
 

cf/include/log.h

@@ -293,5 +293,5 @@ void cf_log_dump_cache(void);
 // Public API - stack trace.
 //
 
-void cf_log_stack_trace(void* ctx);
-uint64_t cf_log_strip_aslr(const void* addr);
+// For debugging, in case we need a stack trace without stopping the server.
+void cf_log_stack_trace(void);

cf/include/trace.h

@@ -0,0 +1,50 @@
+/*
+ * trace.h
+ *
+ * Copyright (C) 2023 Aerospike, Inc.
+ *
+ * Portions may be licensed to Aerospike, Inc. under one or more contributor
+ * license agreements.
+ *
+ * This program is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/
+ */
+
+#pragma once
+
+//==========================================================
+// Includes.
+//
+
+#include <stdint.h>
+
+
+//==========================================================
+// Typedefs & constants.
+//
+
+#define MAX_BACKTRACE_DEPTH 50
+
+// We assume this is big enough (for max path length, max identifier length,
+// line number, overhead).
+#define SYM_STR_MAX_SZ (8 * 1024)
+
+
+//==========================================================
+// Public API.
+//
+
+void cf_trace_init(void);
+int cf_backtrace(void** buf, int sz);
+uint64_t cf_strip_aslr(const void* addr);
+void cf_addr_to_sym_str(char* buf, const void* addr);

cf/src/Makefile

@@ -34,6 +34,7 @@ HEADERS += secrets.h
 HEADERS += shash.h
 HEADERS += socket.h
 HEADERS += tls.h
+HEADERS += trace.h
 HEADERS += uds.h
 HEADERS += vault.h
 HEADERS += vector.h
@@ -62,6 +63,7 @@ SOURCES += pool.c
 SOURCES += rchash.c
 SOURCES += shash.c
 SOURCES += socket.c
+SOURCES += trace.c
 SOURCES += uds.c
 SOURCES += vector.c
 SOURCES += vmapx.c
@@ -81,6 +83,7 @@ INCLUDES += $(INCLUDE_DIR:%=-I%)
 INCLUDES += -I$(COMMON)/target/$(PLATFORM)/include
 INCLUDES += -I$(JANSSON)/src
 INCLUDES += -I$(JEMALLOC)/include
+INCLUDES += -I$(LIBBACKTRACE)
 
 OBJECTS = $(SOURCES:%.c=$(OBJECT_DIR)/%.o)
 DEPENDENCIES = $(OBJECTS:%.o=%.d)

cf/src/alloc.c

@@ -46,6 +46,7 @@
 #include "bits.h"
 #include "cf_thread.h"
 #include "log.h"
+#include "trace.h"
 
 #include "aerospike/as_atomic.h"
 #include "aerospike/as_random.h"
@@ -305,28 +306,28 @@ hook_handle_free_check(const void* ra, const void* p, size_t jem_sz)
 
 	if (site_id >= MAX_SITES) {
 		cf_crash(CF_ALLOC, "corruption %zu@%p RA 0x%lx, invalid site ID",
-				jem_sz, p, cf_log_strip_aslr(ra));
+				jem_sz, p, cf_strip_aslr(ra));
 	}
 
 	if (delta == 0xffff) {
 		cf_crash(CF_ALLOC, "corruption %zu@%p RA 0x%lx, potential double free, possibly freed before with RA 0x%lx",
-				jem_sz, p, cf_log_strip_aslr(ra),
-				cf_log_strip_aslr(as_load_rlx(g_site_ras + site_id)));
+				jem_sz, p, cf_strip_aslr(ra),
+				cf_strip_aslr(as_load_rlx(g_site_ras + site_id)));
 	}
 
 	if (delta > jem_sz - sizeof(uint32_t)) {
 		cf_crash(CF_ALLOC, "corruption %zu@%p RA 0x%lx, invalid delta length, possibly allocated with RA 0x%lx",
-				jem_sz, p, cf_log_strip_aslr(ra),
-				cf_log_strip_aslr(as_load_rlx(g_site_ras + site_id)));
+				jem_sz, p, cf_strip_aslr(ra),
+				cf_strip_aslr(as_load_rlx(g_site_ras + site_id)));
 	}
 
 	uint8_t *mark = data - delta;
 
 	for (uint32_t i = 0; i < 4 && i < delta; ++i) {
 		if (mark[i] != data[i]) {
 			cf_crash(CF_ALLOC, "corruption %zu@%p RA 0x%lx, invalid mark, possibly allocated with RA 0x%lx",
-					jem_sz, p, cf_log_strip_aslr(ra),
-					cf_log_strip_aslr(as_load_rlx(g_site_ras + site_id)));
+					jem_sz, p, cf_strip_aslr(ra),
+					cf_strip_aslr(as_load_rlx(g_site_ras + site_id)));
 		}
 	}
 }
@@ -744,7 +745,7 @@ cf_alloc_log_site_infos(const char *file)
 		const void *ra = as_load_rlx(g_site_ras + site_id);
 
 		fprintf(fh, "0x%016" PRIx64 " %9d 0x%016zx 0x%016zx\n",
-				cf_log_strip_aslr(ra), info->thread_id, info->size_hi,
+				cf_strip_aslr(ra), info->thread_id, info->size_hi,
 				info->size_lo);
 	}