From 2ff0ab10a362bd83bdfba5f295a1864a65d4f639 Mon Sep 17 00:00:00 2001 From: Dmitrii Kuvaiskii Date: Wed, 4 Sep 2024 00:39:16 -0700 Subject: [PATCH] [PAL/Linux-SGX] Remove printing of thread-local SGX statistics Previously, when manifest contained `sgx.enable_stats = true`, Gramine-SGX printed not only the process-wide SGX statistics, but also statistics for each thread. These thread-local SGX statistics only litter the output as (1) there are helper Gramine threads that do not correspond to any application thread, (2) there is no easy way to correlate the printed thread-local stats to application threads/workload execution (as stats simply print the opaque TID of the thread). This commit removes thread-local stats and keeps only process-wide ones. Signed-off-by: Dmitrii Kuvaiskii --- Documentation/manifest-syntax.rst | 6 ++-- Documentation/performance.rst | 21 +++---------- pal/src/host/linux-sgx/host_ocalls.c | 46 +++++++++++++--------------- pal/src/host/linux-sgx/host_thread.c | 18 ++--------- pal/src/host/linux-sgx/pal_tcb.h | 2 +- 5 files changed, 31 insertions(+), 62 deletions(-) diff --git a/Documentation/manifest-syntax.rst b/Documentation/manifest-syntax.rst index 2dece85db1..772ef48ed9 100644 --- a/Documentation/manifest-syntax.rst +++ b/Documentation/manifest-syntax.rst @@ -1269,8 +1269,8 @@ large enough to hold the whole heap area. This option is invalid (i.e. must be ``false``) if specified together with ``sgx.edmm_enable``, as there are no heap pages to pre-fault. -Enabling per-thread and process-wide SGX stats -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Enabling SGX enclave stats +^^^^^^^^^^^^^^^^^^^^^^^^^^ :: @@ -1287,7 +1287,7 @@ This syntax specifies whether to enable SGX enclave-specific statistics: number of EENTERs (corresponds to ECALLs plus returns from OCALLs), number of EEXITs (corresponds to OCALLs plus returns from ECALLs) and number of AEXs (corresponds to interrupts/exceptions/signals during enclave - execution). Prints per-thread and per-process stats. + execution). Prints overall stats at the end of enclave execution. #. Printing the SGX enclave loading time at startup. The enclave loading time includes creating the enclave, adding enclave pages, measuring them and diff --git a/Documentation/performance.rst b/Documentation/performance.rst index 081716cb64..44ad2de509 100644 --- a/Documentation/performance.rst +++ b/Documentation/performance.rst @@ -13,27 +13,21 @@ options relevant for performance benchmarking. Refer to in these examples should not be considered representative and serve only illustration purposes. -Enabling per-thread and process-wide SGX stats ----------------------------------------------- +Enabling SGX enclave stats +-------------------------- See also :ref:`perf` below for installing ``perf``. Enable statistics using ``sgx.enable_stats = true`` manifest option. Now your graminized application correctly reports performance counters. This is useful when using e.g. ``perf stat`` to collect performance statistics. This manifest -option also forces Gramine to dump SGX-related information on each -thread/process exit. Here is an example: +option also forces Gramine to dump SGX-related information on each enclave exit. +Here is an example: :: libos/test/regression$ perf stat gramine-sgx helloworld Hello world (helloworld)! - ----- SGX stats for thread 87219 ----- - # of EENTERs: 224 - # of EEXITs: 192 - # of AEXs: 201 - # of sync signals: 32 - # of async signals: 0 ----- Total SGX stats for process 87219 ----- # of EENTERs: 224 # of EEXITs: 192 @@ -48,13 +42,6 @@ thread/process exit. Here is an example: How to read this output: -#. You can see that one thread was created, with Linux-host (actual) ``TID = - 87219``. This one thread belongs to one process, so they have the same ``TID - = PID`` and the same statistics. If the test application would have e.g. two - threads, then there would be two "SGX stats for thread" outputs, and their - values would sum up to the values reported in the final "Total SGX stats for - process". - #. There are about 200 EENTERs and EEXITs. These are mostly due to OCALLs: recall that every OCALL requires one EEXIT to exit the enclave and one EENTER to re-enter it again. We can conclude that there are about 200 OCALLs. Also, diff --git a/pal/src/host/linux-sgx/host_ocalls.c b/pal/src/host/linux-sgx/host_ocalls.c index eba742e55c..45ac7e3f4a 100644 --- a/pal/src/host/linux-sgx/host_ocalls.c +++ b/pal/src/host/linux-sgx/host_ocalls.c @@ -30,6 +30,24 @@ rpc_queue_t* g_rpc_queue = NULL; /* pointer to untrusted queue */ +static noreturn void process_exit(int exitcode) { + update_sgx_stats(/*do_print=*/true); + +#ifdef DEBUG + sgx_profile_finish(); +#endif + +#ifdef SGX_VTUNE_PROFILE + if (g_vtune_profile_enabled) { + extern void __itt_fini_ittlib(void); + __itt_fini_ittlib(); + } +#endif + + DO_SYSCALL(exit_group, exitcode); + die_or_inf_loop(); +} + static long sgx_ocall_exit(void* args) { struct ocall_exit* ocall_exit_args = args; @@ -41,19 +59,7 @@ static long sgx_ocall_exit(void* args) { /* exit the whole process if exit_group() */ if (ocall_exit_args->is_exitgroup) { - update_and_print_stats(/*process_wide=*/true); -#ifdef DEBUG - sgx_profile_finish(); -#endif - -#ifdef SGX_VTUNE_PROFILE - if (g_vtune_profile_enabled) { - extern void __itt_fini_ittlib(void); - __itt_fini_ittlib(); - } -#endif - DO_SYSCALL(exit_group, (int)ocall_exit_args->exitcode); - die_or_inf_loop(); + process_exit((int)ocall_exit_args->exitcode); } /* otherwise call SGX-related thread reset and exit this thread */ @@ -64,20 +70,10 @@ static long sgx_ocall_exit(void* args) { if (!current_enclave_thread_cnt()) { /* no enclave threads left, kill the whole process */ - update_and_print_stats(/*process_wide=*/true); -#ifdef DEBUG - sgx_profile_finish(); -#endif -#ifdef SGX_VTUNE_PROFILE - if (g_vtune_profile_enabled) { - extern void __itt_fini_ittlib(void); - __itt_fini_ittlib(); - } -#endif - DO_SYSCALL(exit_group, (int)ocall_exit_args->exitcode); - die_or_inf_loop(); + process_exit((int)ocall_exit_args->exitcode); } + update_sgx_stats(/*do_print=*/false); thread_exit((int)ocall_exit_args->exitcode); return 0; } diff --git a/pal/src/host/linux-sgx/host_thread.c b/pal/src/host/linux-sgx/host_thread.c index 6db254b192..e5f89bc52d 100644 --- a/pal/src/host/linux-sgx/host_thread.c +++ b/pal/src/host/linux-sgx/host_thread.c @@ -27,7 +27,7 @@ static size_t g_enclave_thread_num = 0; bool g_sgx_enable_stats = false; /* this function is called only on thread/process exit (never in the middle of thread exec) */ -void update_and_print_stats(bool process_wide) { +void update_sgx_stats(bool do_print) { static atomic_ulong g_eenter_cnt = 0; static atomic_ulong g_eexit_cnt = 0; static atomic_ulong g_aex_cnt = 0; @@ -38,25 +38,13 @@ void update_and_print_stats(bool process_wide) { return; PAL_HOST_TCB* tcb = pal_get_host_tcb(); - - int tid = DO_SYSCALL(gettid); - assert(tid > 0); - log_always("----- SGX stats for thread %d -----\n" - " # of EENTERs: %lu\n" - " # of EEXITs: %lu\n" - " # of AEXs: %lu\n" - " # of sync signals: %lu\n" - " # of async signals: %lu", - tid, tcb->eenter_cnt, tcb->eexit_cnt, tcb->aex_cnt, - tcb->sync_signal_cnt, tcb->async_signal_cnt); - g_eenter_cnt += tcb->eenter_cnt; g_eexit_cnt += tcb->eexit_cnt; g_aex_cnt += tcb->aex_cnt; g_sync_signal_cnt += tcb->sync_signal_cnt; g_async_signal_cnt += tcb->async_signal_cnt; - if (process_wide) { + if (do_print) { int pid = g_host_pid; assert(pid > 0); log_always("----- Total SGX stats for process %d -----\n" @@ -286,8 +274,6 @@ noreturn void thread_exit(int status) { * (by sgx_ocall_exit()) but we keep it here for future proof */ block_async_signals(true); - update_and_print_stats(/*process_wide=*/false); - if (tcb->alt_stack) { stack_t ss; ss.ss_sp = NULL; diff --git a/pal/src/host/linux-sgx/pal_tcb.h b/pal/src/host/linux-sgx/pal_tcb.h index 80e797317f..97c4534583 100644 --- a/pal/src/host/linux-sgx/pal_tcb.h +++ b/pal/src/host/linux-sgx/pal_tcb.h @@ -115,5 +115,5 @@ static inline PAL_HOST_TCB* pal_get_host_tcb(void) { } extern bool g_sgx_enable_stats; -void update_and_print_stats(bool process_wide); +void update_sgx_stats(bool do_print); #endif /* IN_ENCLAVE */