Dump and Reset stats data on demand using SIGUSR1 signal

pal/src/host/linux-sgx/host_exception.c

@@ -15,19 +15,22 @@
                      *      __sigset_t uc_sigmask;
                      */
 
-
+#include <dirent.h>
 #include <linux/signal.h>
 #include <stdbool.h>
 
 #include "api.h"
 #include "cpu.h"
 #include "debug_map.h"
+#include "gdb_integration/sgx_gdb.h"
 #include "host_internal.h"
 #include "pal_rpc_queue.h"
+#include "pal_tcb.h"
 #include "sigreturn.h"
 #include "ucontext.h"
 
 static const int ASYNC_SIGNALS[] = {SIGTERM, SIGCONT};
+static int send_sigusr1_signal_to_children(void);
 
 static int block_signal(int sig, bool block) {
     int how = block ? SIG_BLOCK : SIG_UNBLOCK;
@@ -188,6 +191,64 @@ static void handle_dummy_signal(int signum, siginfo_t* info, struct ucontext* uc
     /* we need this handler to interrupt blocking syscalls in RPC threads */
 }
 
+static int send_sigusr1_signal_to_children(void) {
+    int signal_counter= 0;
+
+    for (size_t i = 1; i < MAX_DBG_THREADS; i++) {
+        int child_tid = ((struct enclave_dbginfo*)DBGINFO_ADDR)->thread_tids[i];
+        if(child_tid > 0) {
+            DO_SYSCALL(tkill, child_tid, SIGUSR1);
+            signal_counter++;
+        }
+    }
+
+    return signal_counter;
+}
+
+static void dump_and_reset_stats(void)
+{
+    static atomic_int no_of_children_visited = 0;
+    static const uint64_t LOOP_ATTEMPTS_MAX = 10000;   /* rather arbitrary */
+
+    if(DO_SYSCALL(gettid) == g_host_pid) {
+        int no_of_children = send_sigusr1_signal_to_children();
+        uint64_t loop_attempts = 0;
+
+        /* Wait here until all the children are done processing the signal. */
+        while((no_of_children) > (__atomic_load_n(&no_of_children_visited, __ATOMIC_ACQUIRE))) {
+            if (loop_attempts == LOOP_ATTEMPTS_MAX) {
+                DO_SYSCALL(sched_yield);
+            } else {
+                loop_attempts++;
+                CPU_RELAX();
+            }
+        }
+
+        update_and_print_stats(/*process_wide=*/true);
+        __atomic_store_n(&no_of_children_visited, 0, __ATOMIC_RELEASE);
+    } else {
+        log_always("----- DUMPTING and RESETTING SGX STATS -----");
+        update_and_print_stats(/*process_wide=*/false);
+        __atomic_fetch_add(&no_of_children_visited, 1, __ATOMIC_ACQ_REL);
+    }
+
+    PAL_HOST_TCB* tcb = pal_get_host_tcb();
+    int ret = pal_host_tcb_reset_stats(tcb);
+    if(ret < 0)
+        return;
+}
+
+static void handle_sigusr1(int signum, siginfo_t* info, struct ucontext* uc) {
+    __UNUSED(signum);
+    __UNUSED(info);
+    __UNUSED(uc);
+
+    if(g_sgx_enable_stats)
+        dump_and_reset_stats();
+
+    return;
+}
+
 int sgx_signal_setup(void) {
     int ret;
 
@@ -236,6 +297,10 @@ int sgx_signal_setup(void) {
     if (ret < 0)
         goto err;
 
+    ret = set_signal_handler(SIGUSR1, handle_sigusr1);
+    if (ret < 0)
+        goto err;
+
     ret = 0;
 err:
     return ret;

pal/src/host/linux-sgx/host_thread.c

@@ -50,11 +50,11 @@ void update_and_print_stats(bool process_wide) {
                tid, tcb->eenter_cnt, tcb->eexit_cnt, tcb->aex_cnt,
                tcb->sync_signal_cnt, tcb->async_signal_cnt);
 
-    g_eenter_cnt       += tcb->eenter_cnt;
-    g_eexit_cnt        += tcb->eexit_cnt;
-    g_aex_cnt          += tcb->aex_cnt;
-    g_sync_signal_cnt  += tcb->sync_signal_cnt;
-    g_async_signal_cnt += tcb->async_signal_cnt;
+    __atomic_fetch_add(&g_eenter_cnt, tcb->eenter_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_eexit_cnt, tcb->eexit_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_aex_cnt, tcb->aex_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_sync_signal_cnt, tcb->sync_signal_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_async_signal_cnt, tcb->async_signal_cnt, __ATOMIC_ACQ_REL);
 
     if (process_wide) {
         int pid = g_host_pid;
@@ -67,9 +67,17 @@ void update_and_print_stats(bool process_wide) {
                    "  # of async signals:  %lu",
                    pid, g_eenter_cnt, g_eexit_cnt, g_aex_cnt,
                    g_sync_signal_cnt, g_async_signal_cnt);
+
+        __atomic_store_n(&g_eenter_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_eexit_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_aex_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_sync_signal_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_async_signal_cnt, 0, __ATOMIC_RELEASE);
     }
 }
 
+
+
 void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack) {
     tcb->self = tcb;
     tcb->tcs = NULL;    /* initialized by child thread */
@@ -87,6 +95,23 @@ void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack) {
     tcb->last_async_event = PAL_EVENT_NO_EVENT;
 }
 
+int pal_host_tcb_reset_stats(PAL_HOST_TCB* tcb) {
+    tcb->eenter_cnt       = 0;
+    tcb->eexit_cnt        = 0;
+    tcb->aex_cnt          = 0;
+    tcb->sync_signal_cnt  = 0;
+    tcb->async_signal_cnt = 0;
+
+    int ret;
+
+    ret = DO_SYSCALL(arch_prctl, ARCH_SET_GS, tcb);
+    if (ret < 0) {
+        ret = -EPERM;
+        log_always("error at pal_thread_reset_stats %d", ret);
+    }
+    return ret;
+}
+
 int create_tcs_mapper(void* tcs_base, unsigned int thread_num) {
     sgx_arch_tcs_t* enclave_tcs = tcs_base;
 

pal/src/host/linux-sgx/pal_tcb.h

@@ -108,6 +108,7 @@ typedef struct pal_host_tcb {
 } PAL_HOST_TCB;
 
 extern void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack);
+extern int pal_host_tcb_reset_stats(PAL_HOST_TCB* tcb);
 
 static inline PAL_HOST_TCB* pal_get_host_tcb(void) {
     PAL_HOST_TCB* tcb;