[PAL/Linux-SGX] Remove printing of thread-local SGX statistics

Previously, when manifest contained `sgx.enable_stats = true`,
Gramine-SGX printed not only the process-wide SGX statistics, but also
statistics for each thread. These thread-local SGX statistics only
litter the output as (1) there are helper Gramine threads that do not
correspond to any application thread, (2) there is no easy way to
correlate the printed thread-local stats to application threads/workload
execution (as stats simply print the opaque TID of the thread).

This commit removes thread-local stats and keeps only process-wide ones.

Signed-off-by: Dmitrii Kuvaiskii <[email protected]>

Documentation/manifest-syntax.rst

@@ -1269,8 +1269,8 @@ large enough to hold the whole heap area.
 This option is invalid (i.e. must be ``false``) if specified together with
 ``sgx.edmm_enable``, as there are no heap pages to pre-fault.
 
-Enabling per-thread and process-wide SGX stats
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Enabling SGX enclave stats
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 ::
 
@@ -1287,7 +1287,7 @@ This syntax specifies whether to enable SGX enclave-specific statistics:
    number of EENTERs (corresponds to ECALLs plus returns from OCALLs), number
    of EEXITs (corresponds to OCALLs plus returns from ECALLs) and number of
    AEXs (corresponds to interrupts/exceptions/signals during enclave
-   execution). Prints per-thread and per-process stats.
+   execution). Prints overall stats at the end of enclave execution.
 
 #. Printing the SGX enclave loading time at startup. The enclave loading time
    includes creating the enclave, adding enclave pages, measuring them and

Documentation/performance.rst

@@ -13,27 +13,21 @@ options relevant for performance benchmarking. Refer to
 in these examples should not be considered representative and serve only
 illustration purposes.
 
-Enabling per-thread and process-wide SGX stats
-----------------------------------------------
+Enabling SGX enclave stats
+--------------------------
 
 See also :ref:`perf` below for installing ``perf``.
 
 Enable statistics using ``sgx.enable_stats = true`` manifest option. Now your
 graminized application correctly reports performance counters. This is useful
 when using e.g. ``perf stat`` to collect performance statistics. This manifest
-option also forces Gramine to dump SGX-related information on each
-thread/process exit. Here is an example:
+option also forces Gramine to dump SGX-related information on each enclave exit.
+Here is an example:
 
 ::
 
    libos/test/regression$ perf stat gramine-sgx helloworld
    Hello world (helloworld)!
-   ----- SGX stats for thread 87219 -----
-   # of EENTERs:        224
-   # of EEXITs:         192
-   # of AEXs:           201
-   # of sync signals:   32
-   # of async signals:  0
    ----- Total SGX stats for process 87219 -----
    # of EENTERs:        224
    # of EEXITs:         192
@@ -48,13 +42,6 @@ thread/process exit. Here is an example:
 
 How to read this output:
 
-#. You can see that one thread was created, with Linux-host (actual) ``TID =
-   87219``. This one thread belongs to one process, so they have the same ``TID
-   = PID`` and the same statistics. If the test application would have e.g. two
-   threads, then there would be two "SGX stats for thread" outputs, and their
-   values would sum up to the values reported in the final "Total SGX stats for
-   process".
-
 #. There are about 200 EENTERs and EEXITs. These are mostly due to OCALLs:
    recall that every OCALL requires one EEXIT to exit the enclave and one EENTER
    to re-enter it again. We can conclude that there are about 200 OCALLs. Also,

pal/src/host/linux-sgx/host_ocalls.c

@@ -30,6 +30,24 @@
 
 rpc_queue_t* g_rpc_queue = NULL; /* pointer to untrusted queue */
 
+static noreturn void process_exit(int exitcode) {
+    update_sgx_stats(/*do_print=*/true);
+
+#ifdef DEBUG
+    sgx_profile_finish();
+#endif
+
+#ifdef SGX_VTUNE_PROFILE
+    if (g_vtune_profile_enabled) {
+        extern void __itt_fini_ittlib(void);
+        __itt_fini_ittlib();
+    }
+#endif
+
+    DO_SYSCALL(exit_group, exitcode);
+    die_or_inf_loop();
+}
+
 static long sgx_ocall_exit(void* args) {
     struct ocall_exit* ocall_exit_args = args;
 
@@ -41,19 +59,7 @@ static long sgx_ocall_exit(void* args) {
 
     /* exit the whole process if exit_group() */
     if (ocall_exit_args->is_exitgroup) {
-        update_and_print_stats(/*process_wide=*/true);
-#ifdef DEBUG
-        sgx_profile_finish();
-#endif
-
-#ifdef SGX_VTUNE_PROFILE
-        if (g_vtune_profile_enabled) {
-            extern void __itt_fini_ittlib(void);
-            __itt_fini_ittlib();
-        }
-#endif
-        DO_SYSCALL(exit_group, (int)ocall_exit_args->exitcode);
-        die_or_inf_loop();
+        process_exit((int)ocall_exit_args->exitcode);
     }
 
     /* otherwise call SGX-related thread reset and exit this thread */
@@ -64,20 +70,10 @@ static long sgx_ocall_exit(void* args) {
 
     if (!current_enclave_thread_cnt()) {
         /* no enclave threads left, kill the whole process */
-        update_and_print_stats(/*process_wide=*/true);
-#ifdef DEBUG
-        sgx_profile_finish();
-#endif
-#ifdef SGX_VTUNE_PROFILE
-        if (g_vtune_profile_enabled) {
-            extern void __itt_fini_ittlib(void);
-            __itt_fini_ittlib();
-        }
-#endif
-        DO_SYSCALL(exit_group, (int)ocall_exit_args->exitcode);
-        die_or_inf_loop();
+        process_exit((int)ocall_exit_args->exitcode);
     }
 
+    update_sgx_stats(/*do_print=*/false);
     thread_exit((int)ocall_exit_args->exitcode);
     return 0;
 }

pal/src/host/linux-sgx/host_thread.c

@@ -27,7 +27,7 @@ static size_t g_enclave_thread_num = 0;
 bool g_sgx_enable_stats = false;
 
 /* this function is called only on thread/process exit (never in the middle of thread exec) */
-void update_and_print_stats(bool process_wide) {
+void update_sgx_stats(bool do_print) {
     static atomic_ulong g_eenter_cnt       = 0;
     static atomic_ulong g_eexit_cnt        = 0;
     static atomic_ulong g_aex_cnt          = 0;
@@ -38,25 +38,13 @@ void update_and_print_stats(bool process_wide) {
         return;
 
     PAL_HOST_TCB* tcb = pal_get_host_tcb();
-
-    int tid = DO_SYSCALL(gettid);
-    assert(tid > 0);
-    log_always("----- SGX stats for thread %d -----\n"
-               "  # of EENTERs:        %lu\n"
-               "  # of EEXITs:         %lu\n"
-               "  # of AEXs:           %lu\n"
-               "  # of sync signals:   %lu\n"
-               "  # of async signals:  %lu",
-               tid, tcb->eenter_cnt, tcb->eexit_cnt, tcb->aex_cnt,
-               tcb->sync_signal_cnt, tcb->async_signal_cnt);
-
     g_eenter_cnt       += tcb->eenter_cnt;
     g_eexit_cnt        += tcb->eexit_cnt;
     g_aex_cnt          += tcb->aex_cnt;
     g_sync_signal_cnt  += tcb->sync_signal_cnt;
     g_async_signal_cnt += tcb->async_signal_cnt;
 
-    if (process_wide) {
+    if (do_print) {
         int pid = g_host_pid;
         assert(pid > 0);
         log_always("----- Total SGX stats for process %d -----\n"
@@ -286,8 +274,6 @@ noreturn void thread_exit(int status) {
      * (by sgx_ocall_exit()) but we keep it here for future proof */
     block_async_signals(true);
 
-    update_and_print_stats(/*process_wide=*/false);
-
     if (tcb->alt_stack) {
         stack_t ss;
         ss.ss_sp    = NULL;

pal/src/host/linux-sgx/pal_tcb.h

@@ -115,5 +115,5 @@ static inline PAL_HOST_TCB* pal_get_host_tcb(void) {
 }
 
 extern bool g_sgx_enable_stats;
-void update_and_print_stats(bool process_wide);
+void update_sgx_stats(bool do_print);
 #endif /* IN_ENCLAVE */