diff --git a/Documentation/manifest-syntax.rst b/Documentation/manifest-syntax.rst index 2dece85db1..772ef48ed9 100644 --- a/Documentation/manifest-syntax.rst +++ b/Documentation/manifest-syntax.rst @@ -1269,8 +1269,8 @@ large enough to hold the whole heap area. This option is invalid (i.e. must be ``false``) if specified together with ``sgx.edmm_enable``, as there are no heap pages to pre-fault. -Enabling per-thread and process-wide SGX stats -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Enabling SGX enclave stats +^^^^^^^^^^^^^^^^^^^^^^^^^^ :: @@ -1287,7 +1287,7 @@ This syntax specifies whether to enable SGX enclave-specific statistics: number of EENTERs (corresponds to ECALLs plus returns from OCALLs), number of EEXITs (corresponds to OCALLs plus returns from ECALLs) and number of AEXs (corresponds to interrupts/exceptions/signals during enclave - execution). Prints per-thread and per-process stats. + execution). Prints overall stats at the end of enclave execution. #. Printing the SGX enclave loading time at startup. The enclave loading time includes creating the enclave, adding enclave pages, measuring them and diff --git a/Documentation/performance.rst b/Documentation/performance.rst index 081716cb64..44ad2de509 100644 --- a/Documentation/performance.rst +++ b/Documentation/performance.rst @@ -13,27 +13,21 @@ options relevant for performance benchmarking. Refer to in these examples should not be considered representative and serve only illustration purposes. -Enabling per-thread and process-wide SGX stats ----------------------------------------------- +Enabling SGX enclave stats +-------------------------- See also :ref:`perf` below for installing ``perf``. Enable statistics using ``sgx.enable_stats = true`` manifest option. Now your graminized application correctly reports performance counters. This is useful when using e.g. ``perf stat`` to collect performance statistics. This manifest -option also forces Gramine to dump SGX-related information on each -thread/process exit. Here is an example: +option also forces Gramine to dump SGX-related information on each enclave exit. +Here is an example: :: libos/test/regression$ perf stat gramine-sgx helloworld Hello world (helloworld)! - ----- SGX stats for thread 87219 ----- - # of EENTERs: 224 - # of EEXITs: 192 - # of AEXs: 201 - # of sync signals: 32 - # of async signals: 0 ----- Total SGX stats for process 87219 ----- # of EENTERs: 224 # of EEXITs: 192 @@ -48,13 +42,6 @@ thread/process exit. Here is an example: How to read this output: -#. You can see that one thread was created, with Linux-host (actual) ``TID = - 87219``. This one thread belongs to one process, so they have the same ``TID - = PID`` and the same statistics. If the test application would have e.g. two - threads, then there would be two "SGX stats for thread" outputs, and their - values would sum up to the values reported in the final "Total SGX stats for - process". - #. There are about 200 EENTERs and EEXITs. These are mostly due to OCALLs: recall that every OCALL requires one EEXIT to exit the enclave and one EENTER to re-enter it again. We can conclude that there are about 200 OCALLs. Also, diff --git a/pal/src/host/linux-sgx/host_ocalls.c b/pal/src/host/linux-sgx/host_ocalls.c index eba742e55c..45ac7e3f4a 100644 --- a/pal/src/host/linux-sgx/host_ocalls.c +++ b/pal/src/host/linux-sgx/host_ocalls.c @@ -30,6 +30,24 @@ rpc_queue_t* g_rpc_queue = NULL; /* pointer to untrusted queue */ +static noreturn void process_exit(int exitcode) { + update_sgx_stats(/*do_print=*/true); + +#ifdef DEBUG + sgx_profile_finish(); +#endif + +#ifdef SGX_VTUNE_PROFILE + if (g_vtune_profile_enabled) { + extern void __itt_fini_ittlib(void); + __itt_fini_ittlib(); + } +#endif + + DO_SYSCALL(exit_group, exitcode); + die_or_inf_loop(); +} + static long sgx_ocall_exit(void* args) { struct ocall_exit* ocall_exit_args = args; @@ -41,19 +59,7 @@ static long sgx_ocall_exit(void* args) { /* exit the whole process if exit_group() */ if (ocall_exit_args->is_exitgroup) { - update_and_print_stats(/*process_wide=*/true); -#ifdef DEBUG - sgx_profile_finish(); -#endif - -#ifdef SGX_VTUNE_PROFILE - if (g_vtune_profile_enabled) { - extern void __itt_fini_ittlib(void); - __itt_fini_ittlib(); - } -#endif - DO_SYSCALL(exit_group, (int)ocall_exit_args->exitcode); - die_or_inf_loop(); + process_exit((int)ocall_exit_args->exitcode); } /* otherwise call SGX-related thread reset and exit this thread */ @@ -64,20 +70,10 @@ static long sgx_ocall_exit(void* args) { if (!current_enclave_thread_cnt()) { /* no enclave threads left, kill the whole process */ - update_and_print_stats(/*process_wide=*/true); -#ifdef DEBUG - sgx_profile_finish(); -#endif -#ifdef SGX_VTUNE_PROFILE - if (g_vtune_profile_enabled) { - extern void __itt_fini_ittlib(void); - __itt_fini_ittlib(); - } -#endif - DO_SYSCALL(exit_group, (int)ocall_exit_args->exitcode); - die_or_inf_loop(); + process_exit((int)ocall_exit_args->exitcode); } + update_sgx_stats(/*do_print=*/false); thread_exit((int)ocall_exit_args->exitcode); return 0; } diff --git a/pal/src/host/linux-sgx/host_thread.c b/pal/src/host/linux-sgx/host_thread.c index 6db254b192..e5f89bc52d 100644 --- a/pal/src/host/linux-sgx/host_thread.c +++ b/pal/src/host/linux-sgx/host_thread.c @@ -27,7 +27,7 @@ static size_t g_enclave_thread_num = 0; bool g_sgx_enable_stats = false; /* this function is called only on thread/process exit (never in the middle of thread exec) */ -void update_and_print_stats(bool process_wide) { +void update_sgx_stats(bool do_print) { static atomic_ulong g_eenter_cnt = 0; static atomic_ulong g_eexit_cnt = 0; static atomic_ulong g_aex_cnt = 0; @@ -38,25 +38,13 @@ void update_and_print_stats(bool process_wide) { return; PAL_HOST_TCB* tcb = pal_get_host_tcb(); - - int tid = DO_SYSCALL(gettid); - assert(tid > 0); - log_always("----- SGX stats for thread %d -----\n" - " # of EENTERs: %lu\n" - " # of EEXITs: %lu\n" - " # of AEXs: %lu\n" - " # of sync signals: %lu\n" - " # of async signals: %lu", - tid, tcb->eenter_cnt, tcb->eexit_cnt, tcb->aex_cnt, - tcb->sync_signal_cnt, tcb->async_signal_cnt); - g_eenter_cnt += tcb->eenter_cnt; g_eexit_cnt += tcb->eexit_cnt; g_aex_cnt += tcb->aex_cnt; g_sync_signal_cnt += tcb->sync_signal_cnt; g_async_signal_cnt += tcb->async_signal_cnt; - if (process_wide) { + if (do_print) { int pid = g_host_pid; assert(pid > 0); log_always("----- Total SGX stats for process %d -----\n" @@ -286,8 +274,6 @@ noreturn void thread_exit(int status) { * (by sgx_ocall_exit()) but we keep it here for future proof */ block_async_signals(true); - update_and_print_stats(/*process_wide=*/false); - if (tcb->alt_stack) { stack_t ss; ss.ss_sp = NULL; diff --git a/pal/src/host/linux-sgx/pal_tcb.h b/pal/src/host/linux-sgx/pal_tcb.h index 80e797317f..97c4534583 100644 --- a/pal/src/host/linux-sgx/pal_tcb.h +++ b/pal/src/host/linux-sgx/pal_tcb.h @@ -115,5 +115,5 @@ static inline PAL_HOST_TCB* pal_get_host_tcb(void) { } extern bool g_sgx_enable_stats; -void update_and_print_stats(bool process_wide); +void update_sgx_stats(bool do_print); #endif /* IN_ENCLAVE */