Add a support for debug-stats dumping and resetting on demand

Signed-off-by: TejaswineeL <[email protected]>

pal/src/host/linux-sgx/host_exception.c

@@ -15,19 +15,27 @@
                      *      __sigset_t uc_sigmask;
                      */
 
-
+#include <dirent.h>
 #include <linux/signal.h>
 #include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
 
 #include "api.h"
 #include "cpu.h"
 #include "debug_map.h"
+#include "gdb_integration/sgx_gdb.h"
 #include "host_internal.h"
 #include "pal_rpc_queue.h"
+#include "pal_tcb.h"
 #include "sigreturn.h"
 #include "ucontext.h"
 
+#define MAX_DBG_THREADS 4096
+
 static const int ASYNC_SIGNALS[] = {SIGTERM, SIGCONT};
+static int send_sigusr1_signal_to_children(void);
 
 static int block_signal(int sig, bool block) {
     int how = block ? SIG_BLOCK : SIG_UNBLOCK;
@@ -188,6 +196,61 @@ static void handle_dummy_signal(int signum, siginfo_t* info, struct ucontext* uc
     /* we need this handler to interrupt blocking syscalls in RPC threads */
 }
 
+static int send_sigusr1_signal_to_children() {
+    int signal_counter= 0;
+
+    for (size_t i = 1; i < MAX_DBG_THREADS; i++) {
+        int child_tid = ((struct enclave_dbginfo*)DBGINFO_ADDR)->thread_tids[i];
+        if(child_tid > 0) {
+            DO_SYSCALL(tkill, child_tid, SIGUSR1);
+            signal_counter++;
+        }
+    }
+    return signal_counter;
+}
+
+static void handle_async_sigusr1_signal(int signum, siginfo_t* info, struct ucontext* uc) {
+    __UNUSED(signum);
+    __UNUSED(info);
+    __UNUSED(uc);
+
+    static atomic_int no_of_children_visited = 0;
+    static const uint64_t LOOP_ATTEMPTS_MAX = 10000;   /* rather arbitrary */
+    static const uint64_t SLEEP_MAX    = 100000000; /* nanoseconds (0.1 seconds) */
+    static const uint64_t SLEEP_STEP   = 1000000;   /* 100 steps before capped */
+
+    if(g_sgx_enable_stats) {
+
+        if(DO_SYSCALL(gettid) == g_host_pid) {
+            int no_of_children = send_sigusr1_signal_to_children();
+            uint64_t loop_attempts = 0;
+            uint64_t sleep_time    = 0;
+
+            while((no_of_children) > (__atomic_load_n(&no_of_children_visited, __ATOMIC_RELAXED))) {
+                if (loop_attempts == LOOP_ATTEMPTS_MAX) {
+                    if (sleep_time < SLEEP_MAX)
+                        sleep_time += SLEEP_STEP;
+                    struct timespec tv = {.tv_sec = 0, .tv_nsec = sleep_time};
+                    (void)DO_SYSCALL(nanosleep, &tv, /*rem=*/NULL);
+                } else {
+                        loop_attempts++;
+                        CPU_RELAX();
+                }
+            }
+            update_and_print_stats(true);
+            __atomic_exchange_n(&no_of_children_visited, 0, __ATOMIC_ACQ_REL);
+        } else {
+            log_always("----- DUMPTING and RESETTING SGX STATS -----");
+            update_and_print_stats(/*process_wide=*/false);
+            PAL_HOST_TCB* tcb = pal_get_host_tcb();
+            int ret = pal_host_tcb_reset_stats(tcb);
+            if(ret < 0)
+                return;
+            __atomic_fetch_add(&no_of_children_visited, 1, __ATOMIC_ACQ_REL);
+        }
+    }
+}
+
 int sgx_signal_setup(void) {
     int ret;
 
@@ -236,6 +299,10 @@ int sgx_signal_setup(void) {
     if (ret < 0)
         goto err;
 
+    ret = set_signal_handler(SIGUSR1, handle_async_sigusr1_signal);
+    if (ret < 0)
+        goto err;
+
     ret = 0;
 err:
     return ret;

pal/src/host/linux-sgx/host_thread.c

@@ -50,11 +50,11 @@ void update_and_print_stats(bool process_wide) {
                tid, tcb->eenter_cnt, tcb->eexit_cnt, tcb->aex_cnt,
                tcb->sync_signal_cnt, tcb->async_signal_cnt);
 
-    g_eenter_cnt       += tcb->eenter_cnt;
-    g_eexit_cnt        += tcb->eexit_cnt;
-    g_aex_cnt          += tcb->aex_cnt;
-    g_sync_signal_cnt  += tcb->sync_signal_cnt;
-    g_async_signal_cnt += tcb->async_signal_cnt;
+    __atomic_fetch_add(&g_eenter_cnt, tcb->eenter_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_eexit_cnt, tcb->eexit_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_aex_cnt, tcb->aex_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_sync_signal_cnt, tcb->sync_signal_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_async_signal_cnt, tcb->async_signal_cnt, __ATOMIC_ACQ_REL);
 
     if (process_wide) {
         int pid = g_host_pid;
@@ -67,9 +67,17 @@ void update_and_print_stats(bool process_wide) {
                    "  # of async signals:  %lu",
                    pid, g_eenter_cnt, g_eexit_cnt, g_aex_cnt,
                    g_sync_signal_cnt, g_async_signal_cnt);
+
+        __atomic_exchange_n(&g_eenter_cnt, 0, __ATOMIC_ACQ_REL);
+        __atomic_exchange_n(&g_eexit_cnt, 0, __ATOMIC_ACQ_REL);
+        __atomic_exchange_n(&g_aex_cnt, 0, __ATOMIC_ACQ_REL);
+        __atomic_exchange_n(&g_sync_signal_cnt, 0, __ATOMIC_ACQ_REL);
+        __atomic_exchange_n(&g_async_signal_cnt, 0, __ATOMIC_ACQ_REL);
     }
 }
 
+
+
 void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack) {
     tcb->self = tcb;
     tcb->tcs = NULL;    /* initialized by child thread */
@@ -87,6 +95,24 @@ void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack) {
     tcb->last_async_event = PAL_EVENT_NO_EVENT;
 }
 
+int pal_host_tcb_reset_stats(PAL_HOST_TCB* tcb) {
+    tcb->eenter_cnt       = 0;
+    tcb->eexit_cnt        = 0;
+    tcb->aex_cnt          = 0;
+    tcb->sync_signal_cnt  = 0;
+    tcb->async_signal_cnt = 0;
+
+    int ret;
+
+    /* set GS reg of this thread to thread's TCB; */
+    ret = DO_SYSCALL(arch_prctl, ARCH_SET_GS, tcb);
+    if (ret < 0) {
+        ret = -EPERM;
+        log_always("error at pal_thread_reset_stats %d", ret);
+    }
+    return ret;
+}
+
 int create_tcs_mapper(void* tcs_base, unsigned int thread_num) {
     sgx_arch_tcs_t* enclave_tcs = tcs_base;
 

pal/src/host/linux-sgx/pal_tcb.h

@@ -108,6 +108,7 @@ typedef struct pal_host_tcb {
 } PAL_HOST_TCB;
 
 extern void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack);
+extern int pal_host_tcb_reset_stats(PAL_HOST_TCB* tcb);
 
 static inline PAL_HOST_TCB* pal_get_host_tcb(void) {
     PAL_HOST_TCB* tcb;