From a0740847ed6d11281399e59707629210868d388f Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Thu, 22 Feb 2024 16:02:47 +0530 Subject: [PATCH 01/10] Page Cache support for trusted files Signed-off-by: jkr0103 --- CI-Examples/nginx/nginx-gramine.conf.template | 13 +- CI-Examples/nginx/nginx.manifest.template | 3 + Documentation/manifest-syntax.rst | 14 ++ Documentation/performance.rst | 13 ++ .../protected_files => include}/lru_cache.h | 0 common/src/{protected_files => }/lru_cache.c | 0 common/src/meson.build | 1 + common/src/protected_files/meson.build | 8 +- pal/src/host/linux-sgx/enclave_framework.c | 144 +++++++++++++----- pal/src/host/linux-sgx/enclave_tf.h | 12 ++ pal/src/host/linux-sgx/enclave_tf_structs.h | 9 ++ pal/src/host/linux-sgx/pal_files.c | 2 + pal/src/host/linux-sgx/pal_main.c | 12 ++ 13 files changed, 189 insertions(+), 42 deletions(-) rename common/{src/protected_files => include}/lru_cache.h (100%) rename common/src/{protected_files => }/lru_cache.c (100%) diff --git a/CI-Examples/nginx/nginx-gramine.conf.template b/CI-Examples/nginx/nginx-gramine.conf.template index 2f4bfbcf1b..f83a37d2bb 100644 --- a/CI-Examples/nginx/nginx-gramine.conf.template +++ b/CI-Examples/nginx/nginx-gramine.conf.template @@ -17,7 +17,7 @@ # Uncomment "user nobody;" below to switch to this user. If you run under root, use # "user root;" instead. Typically there is no need to specify a non-default user. #user nobody; -worker_processes 4; +worker_processes auto; #error_log logs/error.log; #error_log logs/error.log notice; @@ -30,6 +30,17 @@ events { } http { + access_log off; + client_body_buffer_size 80k; + client_max_body_size 9m; + client_header_buffer_size 1k; + client_body_timeout 10; + client_header_timeout 10; + send_timeout 10; + open_file_cache max=1024 inactive=10s; + open_file_cache_valid 60s; + open_file_cache_min_uses 2; + open_file_cache_errors on; include mime.types; default_type application/octet-stream; sendfile on; diff --git a/CI-Examples/nginx/nginx.manifest.template b/CI-Examples/nginx/nginx.manifest.template index 994b770396..1b883d8392 100644 --- a/CI-Examples/nginx/nginx.manifest.template +++ b/CI-Examples/nginx/nginx.manifest.template @@ -33,6 +33,9 @@ sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }} sgx.enclave_size = "512M" sgx.max_threads = {{ '1' if env.get('EDMM', '0') == '1' else '4' }} +# Tune this parameters based on your application and system configuration for best performance +sgx.tf_max_chunks_in_cache = 48 + sgx.trusted_files = [ "file:{{ gramine.libos }}", "file:{{ install_dir }}/sbin/nginx", diff --git a/Documentation/manifest-syntax.rst b/Documentation/manifest-syntax.rst index 5aa0e27a5a..417b391414 100644 --- a/Documentation/manifest-syntax.rst +++ b/Documentation/manifest-syntax.rst @@ -968,6 +968,20 @@ Marking files as trusted is especially useful for shared libraries: a |~| trusted library cannot be silently replaced by a malicious host because the hash verification will fail. +.. _trusted-files-max-chunks-in-cache: + +Trusted files Page Cache optimization: maximum chunks in cache +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + sgx.tf_max_chunks_in_cache = [NUM] + (Default: 0) + +This syntax specifies a limit on the number of chunks in the cache for trusted +files. By default, this optimization is disabled as the limit should be +application-specific. + .. _encrypted-files: Encrypted files diff --git a/Documentation/performance.rst b/Documentation/performance.rst index 515d1d335b..3882eba44b 100644 --- a/Documentation/performance.rst +++ b/Documentation/performance.rst @@ -305,6 +305,19 @@ in Gramine: all IPC is transparently encrypted/decrypted using the TLS-PSK with AES-GCM crypto. +Cache optimization for trusted files +------------------------------------ + +As the trusted file is read, it is split into chunks, and we compute SHA256 +hash for each chunk. Gramine doesn't implement a Page Cache feature, so it +doesn't have the optimization of keeping the trusted file in enclave memory. +To address this performance bottleneck, instead of loading file chunks in the +enclave each time the file is read, the file chunks are kept in cache. This +optimization is by default disabled, but can be enabled and tuned according to +the needs of the application via the manifest option +``sgx.tf_max_chunks_in_cache``. See :ref:`trusted-files-max-chunks-in-cache` +for more information. + .. _choice_of_sgx_machine: Choice of SGX machine diff --git a/common/src/protected_files/lru_cache.h b/common/include/lru_cache.h similarity index 100% rename from common/src/protected_files/lru_cache.h rename to common/include/lru_cache.h diff --git a/common/src/protected_files/lru_cache.c b/common/src/lru_cache.c similarity index 100% rename from common/src/protected_files/lru_cache.c rename to common/src/lru_cache.c diff --git a/common/src/meson.build b/common/src/meson.build index fa7c0851f1..0c32eea5b4 100644 --- a/common/src/meson.build +++ b/common/src/meson.build @@ -7,6 +7,7 @@ common_src_internal = files( 'avl_tree.c', 'init.c', 'location.c', + 'lru_cache.c', 'pal_error.c', 'printf.c', 'socket_utils.c', diff --git a/common/src/protected_files/meson.build b/common/src/protected_files/meson.build index d604050ab9..52ef35914d 100644 --- a/common/src/protected_files/meson.build +++ b/common/src/protected_files/meson.build @@ -1,14 +1,15 @@ protected_files_inc = [ - include_directories('.'), + include_directories('.', + '../../../pal/include/pal', + '../../../pal/include/arch' / host_machine.cpu_family(), + '../../../pal/include/arch' / host_machine.cpu_family() / 'linux'), common_inc, ] protected_files_dep = declare_dependency( sources: [ - 'lru_cache.c', 'protected_files.c', - 'lru_cache.h', 'protected_files_format.h', 'protected_files.h', 'protected_files_internal.h', @@ -18,5 +19,6 @@ protected_files_dep = declare_dependency( dependencies: [ uthash_dep, + common_dep, ] ) diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index ed9398ed78..b950c3f8c6 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -23,6 +23,7 @@ static int register_file(const char* uri, const char* hash_str, bool check_dupli uintptr_t g_enclave_base; uintptr_t g_enclave_top; bool g_allowed_files_warn = false; +size_t g_tf_max_chunks_in_cache = 0; /* * SGX's EGETKEY(SEAL_KEY) uses three masks as key-derivation material: @@ -624,6 +625,41 @@ static void set_file_check_policy(int policy) { g_file_check_policy = policy; } +int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, + uint64_t chunk_size, uint64_t chunk_number) { + if (chunk_number == 0) { + tf->times_first_chunk_loaded++; + } + + if (tf->times_first_chunk_loaded > 1) { + tf_chunk_t* new_chunk = calloc(1, sizeof(*new_chunk)); + if (!new_chunk) { + return -PAL_ERROR_NOMEM; + } + new_chunk->chunk_number = chunk_number; + memcpy(new_chunk->data, chunk, chunk_size); + + if (!lruc_add(tf->cache, chunk_number, new_chunk)) { + free(new_chunk); + return -PAL_ERROR_NOMEM; + } + if (lruc_size(tf->cache) > (size_t)g_tf_max_chunks_in_cache) { + free(lruc_get_last(tf->cache)); + lruc_remove_last(tf->cache); +#ifdef DEBUG + static int lcu_remove_count = 0; + if (g_tf_max_chunks_in_cache > 0 && ++lcu_remove_count == 100) { + log_always("High frequenty of this log indicates Trusted files chunks exceed the" + " `tf_max_chunks_in_cache` limit. Please increase it in the manifest" + " file to get the best performance."); + lcu_remove_count = 0; + } +#endif /* DEBUG */ + } + } + return 0; +} + int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* umem, off_t aligned_offset, off_t aligned_end, off_t offset, off_t end, sgx_chunk_hash_t* chunk_hashes, size_t file_size) { @@ -642,59 +678,91 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume uint8_t* buf_pos = buf; off_t chunk_offset = aligned_offset; - for (; chunk_offset < aligned_end; chunk_offset += TRUSTED_CHUNK_SIZE, chunk_hashes_item++) { + + struct trusted_file* tf = get_trusted_or_allowed_file(path); + int chunk_number = chunk_offset/TRUSTED_CHUNK_SIZE; + + for (; chunk_offset < aligned_end; chunk_offset += TRUSTED_CHUNK_SIZE, chunk_hashes_item++, chunk_number++) { size_t chunk_size = MIN(file_size - chunk_offset, TRUSTED_CHUNK_SIZE); off_t chunk_end = chunk_offset + chunk_size; - sgx_chunk_hash_t chunk_hash[2]; /* each chunk_hash is 128 bits in size but we need 256 */ + if (lruc_find(tf->cache, chunk_number) != NULL) { + tf_chunk_t* chunk = lruc_get(tf->cache, chunk_number); - LIB_SHA256_CONTEXT chunk_sha; - ret = lib_SHA256Init(&chunk_sha); - if (ret < 0) - goto failed; + if (chunk_offset >= offset && chunk_end <= end) { + memcpy(buf_pos, chunk->data, chunk_size); - if (chunk_offset >= offset && chunk_end <= end) { - /* if current chunk-to-copy completely resides in the requested region-to-copy, - * directly copy into buf (without a scratch buffer) and hash in-place */ - if (!sgx_copy_to_enclave(buf_pos, chunk_size, umem + chunk_offset, chunk_size)) { - goto failed; + buf_pos += chunk_size; + } else { + memcpy(tmp_chunk, chunk->data, chunk_size); + + off_t copy_start = MAX(chunk_offset, offset); + off_t copy_end = MIN(chunk_offset + (off_t)chunk_size, end); + assert(copy_end > copy_start); + + memcpy(buf_pos, tmp_chunk + copy_start - chunk_offset, copy_end - copy_start); + buf_pos += copy_end - copy_start; } + } + else { + sgx_chunk_hash_t chunk_hash[2]; /* each chunk_hash is 128 bits in size but we need 256 */ - ret = lib_SHA256Update(&chunk_sha, buf_pos, chunk_size); + LIB_SHA256_CONTEXT chunk_sha; + ret = lib_SHA256Init(&chunk_sha); if (ret < 0) goto failed; - buf_pos += chunk_size; - } else { - /* if current chunk-to-copy only partially overlaps with the requested region-to-copy, - * read the file contents into a scratch buffer, verify hash and then copy only the part - * needed by the caller */ - if (!sgx_copy_to_enclave(tmp_chunk, chunk_size, umem + chunk_offset, chunk_size)) { - goto failed; + if (chunk_offset >= offset && chunk_end <= end) { + /* if current chunk-to-copy completely resides in the requested region-to-copy, + * directly copy into buf (without a scratch buffer) and hash in-place */ + if (!sgx_copy_to_enclave(buf_pos, chunk_size, umem + chunk_offset, chunk_size)) { + goto failed; + } + + ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); + if (ret < 0) + goto failed; + + ret = lib_SHA256Update(&chunk_sha, buf_pos, chunk_size); + if (ret < 0) + goto failed; + + buf_pos += chunk_size; + } else { + /* if current chunk-to-copy only partially overlaps with the requested region-to-copy, + * read the file contents into a scratch buffer, verify hash and then copy only the part + * needed by the caller */ + if (!sgx_copy_to_enclave(tmp_chunk, chunk_size, umem + chunk_offset, chunk_size)) { + goto failed; + } + + ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); + if (ret < 0) + goto failed; + + ret = lib_SHA256Update(&chunk_sha, tmp_chunk, chunk_size); + if (ret < 0) + goto failed; + + /* determine which part of the chunk is needed by the caller */ + off_t copy_start = MAX(chunk_offset, offset); + off_t copy_end = MIN(chunk_offset + (off_t)chunk_size, end); + assert(copy_end > copy_start); + + memcpy(buf_pos, tmp_chunk + copy_start - chunk_offset, copy_end - copy_start); + buf_pos += copy_end - copy_start; } - ret = lib_SHA256Update(&chunk_sha, tmp_chunk, chunk_size); + ret = lib_SHA256Final(&chunk_sha, (uint8_t*)&chunk_hash[0]); if (ret < 0) goto failed; - /* determine which part of the chunk is needed by the caller */ - off_t copy_start = MAX(chunk_offset, offset); - off_t copy_end = MIN(chunk_offset + (off_t)chunk_size, end); - assert(copy_end > copy_start); - - memcpy(buf_pos, tmp_chunk + copy_start - chunk_offset, copy_end - copy_start); - buf_pos += copy_end - copy_start; - } - - ret = lib_SHA256Final(&chunk_sha, (uint8_t*)&chunk_hash[0]); - if (ret < 0) - goto failed; - - if (memcmp(chunk_hashes_item, &chunk_hash[0], sizeof(*chunk_hashes_item))) { - log_error("Accessing file '%s' is denied: incorrect hash of file chunk at %lu-%lu.", - path, chunk_offset, chunk_end); - ret = -PAL_ERROR_DENIED; - goto failed; + if (memcmp(chunk_hashes_item, &chunk_hash[0], sizeof(*chunk_hashes_item))) { + log_error("Accessing file '%s' is denied: incorrect hash of file chunk at %lu-%lu.", + path, chunk_offset, chunk_end); + ret = -PAL_ERROR_DENIED; + goto failed; + } } } diff --git a/pal/src/host/linux-sgx/enclave_tf.h b/pal/src/host/linux-sgx/enclave_tf.h index c930a37aa9..d283f48ed1 100644 --- a/pal/src/host/linux-sgx/enclave_tf.h +++ b/pal/src/host/linux-sgx/enclave_tf.h @@ -78,3 +78,15 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume int init_trusted_files(void); int init_allowed_files(void); + +/*! + * \brief Add trusted file chunk to the cache + * \param tf Trusted file structure. + * \param chunk Trusted file chunk data. + * \param chunk_size Trusted file chunk size. + * \param chunk_number Trusted file chunk number. + * + * \returns 0 on success, negative error code on failure + */ +int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, + uint64_t chunk_size, uint64_t chunk_number); diff --git a/pal/src/host/linux-sgx/enclave_tf_structs.h b/pal/src/host/linux-sgx/enclave_tf_structs.h index f591b38bf7..575e7d34dd 100644 --- a/pal/src/host/linux-sgx/enclave_tf_structs.h +++ b/pal/src/host/linux-sgx/enclave_tf_structs.h @@ -8,6 +8,8 @@ #include #include "list.h" +#include "lru_cache.h" +#include "pal_linux_defs.h" enum { FILE_CHECK_POLICY_STRICT = 0, @@ -35,5 +37,12 @@ struct trusted_file { sgx_file_hash_t file_hash; /* hash over the whole file, retrieved from the manifest */ sgx_chunk_hash_t* chunk_hashes; /* array of hashes over separate file chunks */ size_t uri_len; + lruc_context_t* cache; + uint64_t times_first_chunk_loaded; /* gives a hint how many times a file's content was used */ char uri[]; /* must be NULL-terminated */ }; + +typedef struct _tf_chunk { + uint64_t chunk_number; + uint8_t data[TRUSTED_CHUNK_SIZE]; +} tf_chunk_t; diff --git a/pal/src/host/linux-sgx/pal_files.c b/pal/src/host/linux-sgx/pal_files.c index 0f8c50a868..38c794da43 100644 --- a/pal/src/host/linux-sgx/pal_files.c +++ b/pal/src/host/linux-sgx/pal_files.c @@ -137,6 +137,8 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, /* we lazily update the size of the trusted file */ tf->size = st.st_size; + tf->cache = lruc_create(); + tf->times_first_chunk_loaded = 0; ret = load_trusted_or_allowed_file(tf, hdl, do_create, &chunk_hashes, &total, &umem); if (ret < 0) goto fail; diff --git a/pal/src/host/linux-sgx/pal_main.c b/pal/src/host/linux-sgx/pal_main.c index 3d9cbd867d..ae958bb38e 100644 --- a/pal/src/host/linux-sgx/pal_main.c +++ b/pal/src/host/linux-sgx/pal_main.c @@ -409,6 +409,7 @@ extern void* g_enclave_top; extern bool g_allowed_files_warn; extern uint64_t g_tsc_hz; extern size_t g_unused_tcs_pages_num; +extern int64_t g_tf_max_chunks_in_cache; static int print_warnings_on_insecure_configs(PAL_HANDLE parent_process) { int ret; @@ -864,6 +865,17 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void* ocall_exit(1, /*is_exitgroup=*/true); } + ret = toml_int_in(g_pal_public_state.manifest_root, "sgx.tf_max_chunks_in_cache", + /*defaultval=*/0, &g_tf_max_chunks_in_cache); + if (ret < 0) { + log_error("Cannot parse 'sgx.tf_max_chunks_in_cache'"); + ocall_exit(1, /*is_exitgroup=*/true); + } + if (g_tf_max_chunks_in_cache < 0) { + log_error("Invalid 'sgx.tf_max_chunks_in_cache' value"); + ocall_exit(1, /*is_exitgroup=*/true); + } + ret = toml_bool_in(g_pal_public_state.manifest_root, "sys.enable_extra_runtime_domain_names_conf", /*defaultval*/false, &g_pal_public_state.extra_runtime_domain_names_conf); From 10b9b7352bcb509de2a5cd6e2b6d40ba40105d65 Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Fri, 23 Feb 2024 13:21:37 +0530 Subject: [PATCH 02/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- pal/src/host/linux-sgx/enclave_framework.c | 29 +++++++++++----------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index b950c3f8c6..c01e599f82 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -685,26 +685,21 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume for (; chunk_offset < aligned_end; chunk_offset += TRUSTED_CHUNK_SIZE, chunk_hashes_item++, chunk_number++) { size_t chunk_size = MIN(file_size - chunk_offset, TRUSTED_CHUNK_SIZE); off_t chunk_end = chunk_offset + chunk_size; - - if (lruc_find(tf->cache, chunk_number) != NULL) { - tf_chunk_t* chunk = lruc_get(tf->cache, chunk_number); - + tf_chunk_t* chunk; + if (g_tf_max_chunks_in_cache > 0 && (chunk = lruc_get(tf->cache, chunk_number)) != NULL) { if (chunk_offset >= offset && chunk_end <= end) { memcpy(buf_pos, chunk->data, chunk_size); buf_pos += chunk_size; } else { - memcpy(tmp_chunk, chunk->data, chunk_size); - off_t copy_start = MAX(chunk_offset, offset); off_t copy_end = MIN(chunk_offset + (off_t)chunk_size, end); assert(copy_end > copy_start); - memcpy(buf_pos, tmp_chunk + copy_start - chunk_offset, copy_end - copy_start); + memcpy(buf_pos, chunk->data + copy_start - chunk_offset, copy_end - copy_start); buf_pos += copy_end - copy_start; } - } - else { + } else { sgx_chunk_hash_t chunk_hash[2]; /* each chunk_hash is 128 bits in size but we need 256 */ LIB_SHA256_CONTEXT chunk_sha; @@ -719,9 +714,11 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume goto failed; } - ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); - if (ret < 0) - goto failed; + if(g_tf_max_chunks_in_cache > 0) { + ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); + if (ret < 0) + goto failed; + } ret = lib_SHA256Update(&chunk_sha, buf_pos, chunk_size); if (ret < 0) @@ -736,9 +733,11 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume goto failed; } - ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); - if (ret < 0) - goto failed; + if(g_tf_max_chunks_in_cache > 0) { + ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); + if (ret < 0) + goto failed; + } ret = lib_SHA256Update(&chunk_sha, tmp_chunk, chunk_size); if (ret < 0) From acefe9e6defc4a275b3bb44a1c2a8ca2356036e5 Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Fri, 23 Feb 2024 15:51:50 +0530 Subject: [PATCH 03/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- pal/src/host/linux-sgx/enclave_framework.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index c01e599f82..260b020cfa 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -649,7 +649,7 @@ int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, #ifdef DEBUG static int lcu_remove_count = 0; if (g_tf_max_chunks_in_cache > 0 && ++lcu_remove_count == 100) { - log_always("High frequenty of this log indicates Trusted files chunks exceed the" + log_always("High frequency of this log indicates trusted files chunks exceed the" " `tf_max_chunks_in_cache` limit. Please increase it in the manifest" " file to get the best performance."); lcu_remove_count = 0; @@ -711,6 +711,7 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume /* if current chunk-to-copy completely resides in the requested region-to-copy, * directly copy into buf (without a scratch buffer) and hash in-place */ if (!sgx_copy_to_enclave(buf_pos, chunk_size, umem + chunk_offset, chunk_size)) { + ret = -PAL_ERROR_DENIED; goto failed; } @@ -730,6 +731,7 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume * read the file contents into a scratch buffer, verify hash and then copy only the part * needed by the caller */ if (!sgx_copy_to_enclave(tmp_chunk, chunk_size, umem + chunk_offset, chunk_size)) { + ret = -PAL_ERROR_DENIED; goto failed; } From 7df6e85835f5b0ce7eab49fc6872921ede57d6da Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Fri, 23 Feb 2024 15:53:36 +0530 Subject: [PATCH 04/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- pal/src/host/linux-sgx/enclave_framework.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index 260b020cfa..604e162996 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -650,7 +650,7 @@ int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, static int lcu_remove_count = 0; if (g_tf_max_chunks_in_cache > 0 && ++lcu_remove_count == 100) { log_always("High frequency of this log indicates trusted files chunks exceed the" - " `tf_max_chunks_in_cache` limit. Please increase it in the manifest" + " `sgx.tf_max_chunks_in_cache` limit. Please increase it in the manifest" " file to get the best performance."); lcu_remove_count = 0; } From 15caa0be37cd6726e6a2bd8d4e29d20d75946b31 Mon Sep 17 00:00:00 2001 From: Dmitrii Kuvaiskii Date: Fri, 23 Feb 2024 04:33:08 -0800 Subject: [PATCH 05/10] fixup! Page Cache support for trusted files Signed-off-by: Dmitrii Kuvaiskii --- common/src/meson.build | 14 ++++++++++++++ common/src/protected_files/meson.build | 9 ++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/common/src/meson.build b/common/src/meson.build index 0c32eea5b4..336237cc66 100644 --- a/common/src/meson.build +++ b/common/src/meson.build @@ -3,6 +3,10 @@ common_src_utils = files( 'string_utils.c', ) +common_src_lru_cache = files( + 'lru_cache.c', +) + common_src_internal = files( 'avl_tree.c', 'init.c', @@ -46,6 +50,16 @@ common_utils_dep = declare_dependency( include_directories: common_inc, ) +common_lru_cache_dep = declare_dependency( + sources: common_src_lru_cache, + + include_directories: common_inc, + + dependencies: [ + uthash_dep, + ], +) + common_dep = declare_dependency( sources: common_src_internal, diff --git a/common/src/protected_files/meson.build b/common/src/protected_files/meson.build index 52ef35914d..fbc78737c3 100644 --- a/common/src/protected_files/meson.build +++ b/common/src/protected_files/meson.build @@ -1,15 +1,11 @@ protected_files_inc = [ - include_directories('.', - '../../../pal/include/pal', - '../../../pal/include/arch' / host_machine.cpu_family(), - '../../../pal/include/arch' / host_machine.cpu_family() / 'linux'), + include_directories('.'), common_inc, ] protected_files_dep = declare_dependency( sources: [ 'protected_files.c', - 'protected_files_format.h', 'protected_files.h', 'protected_files_internal.h', @@ -18,7 +14,6 @@ protected_files_dep = declare_dependency( include_directories: protected_files_inc, dependencies: [ - uthash_dep, - common_dep, + common_lru_cache_dep, ] ) From 17658974f684acd38bb68310474a24381550c34d Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Fri, 23 Feb 2024 19:17:41 +0530 Subject: [PATCH 06/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- CI-Examples/nginx/nginx-gramine.conf.template | 13 +-- pal/src/host/linux-sgx/enclave_framework.c | 103 +++++++++--------- pal/src/host/linux-sgx/enclave_tf.h | 12 -- 3 files changed, 53 insertions(+), 75 deletions(-) diff --git a/CI-Examples/nginx/nginx-gramine.conf.template b/CI-Examples/nginx/nginx-gramine.conf.template index f83a37d2bb..2f4bfbcf1b 100644 --- a/CI-Examples/nginx/nginx-gramine.conf.template +++ b/CI-Examples/nginx/nginx-gramine.conf.template @@ -17,7 +17,7 @@ # Uncomment "user nobody;" below to switch to this user. If you run under root, use # "user root;" instead. Typically there is no need to specify a non-default user. #user nobody; -worker_processes auto; +worker_processes 4; #error_log logs/error.log; #error_log logs/error.log notice; @@ -30,17 +30,6 @@ events { } http { - access_log off; - client_body_buffer_size 80k; - client_max_body_size 9m; - client_header_buffer_size 1k; - client_body_timeout 10; - client_header_timeout 10; - send_timeout 10; - open_file_cache max=1024 inactive=10s; - open_file_cache_valid 60s; - open_file_cache_min_uses 2; - open_file_cache_errors on; include mime.types; default_type application/octet-stream; sendfile on; diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index 604e162996..3d84babdba 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -625,7 +625,7 @@ static void set_file_check_policy(int policy) { g_file_check_policy = policy; } -int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, +static int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, uint64_t chunk_size, uint64_t chunk_number) { if (chunk_number == 0) { tf->times_first_chunk_loaded++; @@ -699,71 +699,71 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume memcpy(buf_pos, chunk->data + copy_start - chunk_offset, copy_end - copy_start); buf_pos += copy_end - copy_start; } - } else { - sgx_chunk_hash_t chunk_hash[2]; /* each chunk_hash is 128 bits in size but we need 256 */ - - LIB_SHA256_CONTEXT chunk_sha; - ret = lib_SHA256Init(&chunk_sha); - if (ret < 0) - goto failed; + continue; + } + sgx_chunk_hash_t chunk_hash[2]; /* each chunk_hash is 128 bits in size but we need 256 */ - if (chunk_offset >= offset && chunk_end <= end) { - /* if current chunk-to-copy completely resides in the requested region-to-copy, - * directly copy into buf (without a scratch buffer) and hash in-place */ - if (!sgx_copy_to_enclave(buf_pos, chunk_size, umem + chunk_offset, chunk_size)) { - ret = -PAL_ERROR_DENIED; - goto failed; - } + LIB_SHA256_CONTEXT chunk_sha; + ret = lib_SHA256Init(&chunk_sha); + if (ret < 0) + goto failed; - if(g_tf_max_chunks_in_cache > 0) { - ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); - if (ret < 0) - goto failed; - } + if (chunk_offset >= offset && chunk_end <= end) { + /* if current chunk-to-copy completely resides in the requested region-to-copy, + * directly copy into buf (without a scratch buffer) and hash in-place */ + if (!sgx_copy_to_enclave(buf_pos, chunk_size, umem + chunk_offset, chunk_size)) { + ret = -PAL_ERROR_DENIED; + goto failed; + } - ret = lib_SHA256Update(&chunk_sha, buf_pos, chunk_size); + if(g_tf_max_chunks_in_cache > 0) { + ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); if (ret < 0) goto failed; + } - buf_pos += chunk_size; - } else { - /* if current chunk-to-copy only partially overlaps with the requested region-to-copy, - * read the file contents into a scratch buffer, verify hash and then copy only the part - * needed by the caller */ - if (!sgx_copy_to_enclave(tmp_chunk, chunk_size, umem + chunk_offset, chunk_size)) { - ret = -PAL_ERROR_DENIED; - goto failed; - } + ret = lib_SHA256Update(&chunk_sha, buf_pos, chunk_size); + if (ret < 0) + goto failed; - if(g_tf_max_chunks_in_cache > 0) { - ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); - if (ret < 0) - goto failed; - } + buf_pos += chunk_size; + } else { + /* if current chunk-to-copy only partially overlaps with the requested region-to-copy, + * read the file contents into a scratch buffer, verify hash and then copy only the part + * needed by the caller */ + if (!sgx_copy_to_enclave(tmp_chunk, chunk_size, umem + chunk_offset, chunk_size)) { + ret = -PAL_ERROR_DENIED; + goto failed; + } - ret = lib_SHA256Update(&chunk_sha, tmp_chunk, chunk_size); + if(g_tf_max_chunks_in_cache > 0) { + ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); if (ret < 0) goto failed; - - /* determine which part of the chunk is needed by the caller */ - off_t copy_start = MAX(chunk_offset, offset); - off_t copy_end = MIN(chunk_offset + (off_t)chunk_size, end); - assert(copy_end > copy_start); - - memcpy(buf_pos, tmp_chunk + copy_start - chunk_offset, copy_end - copy_start); - buf_pos += copy_end - copy_start; } - ret = lib_SHA256Final(&chunk_sha, (uint8_t*)&chunk_hash[0]); + ret = lib_SHA256Update(&chunk_sha, tmp_chunk, chunk_size); if (ret < 0) goto failed; - if (memcmp(chunk_hashes_item, &chunk_hash[0], sizeof(*chunk_hashes_item))) { - log_error("Accessing file '%s' is denied: incorrect hash of file chunk at %lu-%lu.", - path, chunk_offset, chunk_end); - ret = -PAL_ERROR_DENIED; - goto failed; - } + /* determine which part of the chunk is needed by the caller */ + off_t copy_start = MAX(chunk_offset, offset); + off_t copy_end = MIN(chunk_offset + (off_t)chunk_size, end); + assert(copy_end > copy_start); + + memcpy(buf_pos, tmp_chunk + copy_start - chunk_offset, copy_end - copy_start); + buf_pos += copy_end - copy_start; + } + + ret = lib_SHA256Final(&chunk_sha, (uint8_t*)&chunk_hash[0]); + if (ret < 0) + goto failed; + + if (memcmp(chunk_hashes_item, &chunk_hash[0], sizeof(*chunk_hashes_item))) { + log_error("Accessing file '%s' is denied: incorrect hash of file chunk at %lu-%lu.", + path, chunk_offset, chunk_end); + ret = -PAL_ERROR_DENIED; + goto failed; } } @@ -771,6 +771,7 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume return 0; failed: + assert(ret < 0); free(tmp_chunk); memset(buf, 0, end - offset); return ret; diff --git a/pal/src/host/linux-sgx/enclave_tf.h b/pal/src/host/linux-sgx/enclave_tf.h index d283f48ed1..c930a37aa9 100644 --- a/pal/src/host/linux-sgx/enclave_tf.h +++ b/pal/src/host/linux-sgx/enclave_tf.h @@ -78,15 +78,3 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume int init_trusted_files(void); int init_allowed_files(void); - -/*! - * \brief Add trusted file chunk to the cache - * \param tf Trusted file structure. - * \param chunk Trusted file chunk data. - * \param chunk_size Trusted file chunk size. - * \param chunk_number Trusted file chunk number. - * - * \returns 0 on success, negative error code on failure - */ -int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, - uint64_t chunk_size, uint64_t chunk_number); From 91641b24c53648552e414610ad9bd53fdc54cea1 Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Fri, 1 Mar 2024 18:06:48 +0530 Subject: [PATCH 07/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- CI-Examples/nginx/nginx.manifest.template | 5 +- Documentation/performance.rst | 16 ++--- common/src/meson.build | 1 - pal/src/host/linux-sgx/enclave_framework.c | 70 +++++++++++++-------- pal/src/host/linux-sgx/enclave_tf.h | 8 ++- pal/src/host/linux-sgx/enclave_tf_structs.h | 6 +- pal/src/host/linux-sgx/meson.build | 1 + pal/src/host/linux-sgx/pal_files.c | 27 +++++--- pal/src/host/linux-sgx/pal_host.h | 1 + pal/src/host/linux-sgx/pal_main.c | 2 +- 10 files changed, 84 insertions(+), 53 deletions(-) diff --git a/CI-Examples/nginx/nginx.manifest.template b/CI-Examples/nginx/nginx.manifest.template index 1b883d8392..94d5db587b 100644 --- a/CI-Examples/nginx/nginx.manifest.template +++ b/CI-Examples/nginx/nginx.manifest.template @@ -33,8 +33,9 @@ sgx.edmm_enable = {{ 'true' if env.get('EDMM', '0') == '1' else 'false' }} sgx.enclave_size = "512M" sgx.max_threads = {{ '1' if env.get('EDMM', '0') == '1' else '4' }} -# Tune this parameters based on your application and system configuration for best performance -sgx.tf_max_chunks_in_cache = 48 +# One trusted-file chunk of 16k size is enough to cover common Nginx static HTTP file of 10k size +# which will be cached by Gramine +sgx.tf_max_chunks_in_cache = 1 sgx.trusted_files = [ "file:{{ gramine.libos }}", diff --git a/Documentation/performance.rst b/Documentation/performance.rst index 3882eba44b..bd47ac6edf 100644 --- a/Documentation/performance.rst +++ b/Documentation/performance.rst @@ -309,14 +309,14 @@ Cache optimization for trusted files ------------------------------------ As the trusted file is read, it is split into chunks, and we compute SHA256 -hash for each chunk. Gramine doesn't implement a Page Cache feature, so it -doesn't have the optimization of keeping the trusted file in enclave memory. -To address this performance bottleneck, instead of loading file chunks in the -enclave each time the file is read, the file chunks are kept in cache. This -optimization is by default disabled, but can be enabled and tuned according to -the needs of the application via the manifest option -``sgx.tf_max_chunks_in_cache``. See :ref:`trusted-files-max-chunks-in-cache` -for more information. +hash for each chunk. Gramine doesn't have an optimization of keeping the trusted +file's content in enclave memory, which implies re-reading and re-hashing the +same file contents every time the file is read at the same offset. To address +this performance bottleneck, instead of loading file chunks in the enclave each +time the file is read, the file chunks are kept in cache. This optimization is +by default disabled, but can be enabled and tuned according to the needs of the +application via the manifest option ``sgx.tf_max_chunks_in_cache``. +See :ref:`trusted-files-max-chunks-in-cache` for more information. .. _choice_of_sgx_machine: diff --git a/common/src/meson.build b/common/src/meson.build index 336237cc66..2f622e1429 100644 --- a/common/src/meson.build +++ b/common/src/meson.build @@ -11,7 +11,6 @@ common_src_internal = files( 'avl_tree.c', 'init.c', 'location.c', - 'lru_cache.c', 'pal_error.c', 'printf.c', 'socket_utils.c', diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index 3d84babdba..e933e1aed8 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -403,7 +403,7 @@ int sgx_get_seal_key(uint16_t key_policy, sgx_key_128bit_t* out_seal_key) { DEFINE_LISTP(trusted_file); static LISTP_TYPE(trusted_file) g_trusted_file_list = LISTP_INIT; -static spinlock_t g_trusted_file_lock = INIT_SPINLOCK_UNLOCKED; +spinlock_t g_trusted_file_lock = INIT_SPINLOCK_UNLOCKED; static int g_file_check_policy = FILE_CHECK_POLICY_STRICT; static void find_path_in_uri(const char* uri, size_t uri_len, const char** out_path, @@ -519,6 +519,10 @@ int load_trusted_or_allowed_file(struct trusted_file* tf, PAL_HANDLE file, bool } spinlock_lock(&g_trusted_file_lock); + if (!(tf->cache = lruc_create())) + return -PAL_ERROR_NOMEM; + tf->usage_count = 0; + if (tf->chunk_hashes) { *out_chunk_hashes = tf->chunk_hashes; spinlock_unlock(&g_trusted_file_lock); @@ -626,41 +630,52 @@ static void set_file_check_policy(int policy) { } static int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, - uint64_t chunk_size, uint64_t chunk_number) { - if (chunk_number == 0) { - tf->times_first_chunk_loaded++; + uint64_t chunk_size, uint64_t chunk_number) { + if (g_tf_max_chunks_in_cache == 0) + return 0; + + // Counts the number of times a file is open and reused + if (chunk_number == 0 && tf->usage_count <= 10) { + spinlock_lock(&g_trusted_file_lock); + tf->usage_count++; + spinlock_unlock(&g_trusted_file_lock); } - if (tf->times_first_chunk_loaded > 1) { - tf_chunk_t* new_chunk = calloc(1, sizeof(*new_chunk)); + // Add file chunks to cache only if the file is reused for 10 times or more + if (tf->usage_count > 10) { + struct tf_chunk* new_chunk = (struct tf_chunk*)malloc(sizeof(struct tf_chunk)); if (!new_chunk) { return -PAL_ERROR_NOMEM; } new_chunk->chunk_number = chunk_number; memcpy(new_chunk->data, chunk, chunk_size); + spinlock_lock(&g_trusted_file_lock); if (!lruc_add(tf->cache, chunk_number, new_chunk)) { free(new_chunk); return -PAL_ERROR_NOMEM; } - if (lruc_size(tf->cache) > (size_t)g_tf_max_chunks_in_cache) { + + if (lruc_size(tf->cache) > g_tf_max_chunks_in_cache) { free(lruc_get_last(tf->cache)); lruc_remove_last(tf->cache); #ifdef DEBUG - static int lcu_remove_count = 0; - if (g_tf_max_chunks_in_cache > 0 && ++lcu_remove_count == 100) { + static int tf_cache_log_throttler = 0; + if (++tf_cache_log_throttler == 100) { log_always("High frequency of this log indicates trusted files chunks exceed the" " `sgx.tf_max_chunks_in_cache` limit. Please increase it in the manifest" " file to get the best performance."); - lcu_remove_count = 0; + tf_cache_log_throttler = 0; } #endif /* DEBUG */ } + spinlock_unlock(&g_trusted_file_lock); + } return 0; } -int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* umem, +int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint8_t* buf, const void* umem, off_t aligned_offset, off_t aligned_end, off_t offset, off_t end, sgx_chunk_hash_t* chunk_hashes, size_t file_size) { int ret = 0; @@ -679,14 +694,18 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume uint8_t* buf_pos = buf; off_t chunk_offset = aligned_offset; - struct trusted_file* tf = get_trusted_or_allowed_file(path); int chunk_number = chunk_offset/TRUSTED_CHUNK_SIZE; for (; chunk_offset < aligned_end; chunk_offset += TRUSTED_CHUNK_SIZE, chunk_hashes_item++, chunk_number++) { size_t chunk_size = MIN(file_size - chunk_offset, TRUSTED_CHUNK_SIZE); off_t chunk_end = chunk_offset + chunk_size; - tf_chunk_t* chunk; - if (g_tf_max_chunks_in_cache > 0 && (chunk = lruc_get(tf->cache, chunk_number)) != NULL) { + struct tf_chunk *chunk; + + spinlock_lock(&g_trusted_file_lock); + chunk = (struct tf_chunk*)lruc_get(tf->cache, chunk_number); + spinlock_unlock(&g_trusted_file_lock); + + if (g_tf_max_chunks_in_cache > 0 && chunk != NULL) { if (chunk_offset >= offset && chunk_end <= end) { memcpy(buf_pos, chunk->data, chunk_size); @@ -701,6 +720,9 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume } continue; } + + /* we didn't find the chunk in the trusted-file cache, must copy into enclave and add to*/ + /* the trusted-file cache */ sgx_chunk_hash_t chunk_hash[2]; /* each chunk_hash is 128 bits in size but we need 256 */ LIB_SHA256_CONTEXT chunk_sha; @@ -710,17 +732,15 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume if (chunk_offset >= offset && chunk_end <= end) { /* if current chunk-to-copy completely resides in the requested region-to-copy, - * directly copy into buf (without a scratch buffer) and hash in-place */ + * directly copy into buf (without a scratch buffer) and hash in-place */ if (!sgx_copy_to_enclave(buf_pos, chunk_size, umem + chunk_offset, chunk_size)) { ret = -PAL_ERROR_DENIED; goto failed; } - if(g_tf_max_chunks_in_cache > 0) { - ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); - if (ret < 0) - goto failed; - } + ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); + if (ret < 0) + goto failed; ret = lib_SHA256Update(&chunk_sha, buf_pos, chunk_size); if (ret < 0) @@ -736,11 +756,9 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume goto failed; } - if(g_tf_max_chunks_in_cache > 0) { - ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); - if (ret < 0) - goto failed; - } + ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); + if (ret < 0) + goto failed; ret = lib_SHA256Update(&chunk_sha, tmp_chunk, chunk_size); if (ret < 0) @@ -761,7 +779,7 @@ int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* ume if (memcmp(chunk_hashes_item, &chunk_hash[0], sizeof(*chunk_hashes_item))) { log_error("Accessing file '%s' is denied: incorrect hash of file chunk at %lu-%lu.", - path, chunk_offset, chunk_end); + path, chunk_offset, chunk_end); ret = -PAL_ERROR_DENIED; goto failed; } diff --git a/pal/src/host/linux-sgx/enclave_tf.h b/pal/src/host/linux-sgx/enclave_tf.h index c930a37aa9..f7410693c7 100644 --- a/pal/src/host/linux-sgx/enclave_tf.h +++ b/pal/src/host/linux-sgx/enclave_tf.h @@ -27,6 +27,7 @@ #include "pal.h" #include "pal_linux_types.h" +extern spinlock_t g_trusted_file_lock; int init_seal_key_material(void); int init_file_check_policy(void); @@ -72,9 +73,10 @@ int load_trusted_or_allowed_file(struct trusted_file* tf, PAL_HANDLE file, bool * * \returns 0 on success, negative error code on failure */ -int copy_and_verify_trusted_file(const char* path, uint8_t* buf, const void* umem, - off_t aligned_offset, off_t aligned_end, off_t offset, off_t end, - sgx_chunk_hash_t* chunk_hashes, size_t file_size); +int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint8_t* buf, + const void* umem, off_t aligned_offset, off_t aligned_end, + off_t offset, off_t end,sgx_chunk_hash_t* chunk_hashes, + size_t file_size); int init_trusted_files(void); int init_allowed_files(void); diff --git a/pal/src/host/linux-sgx/enclave_tf_structs.h b/pal/src/host/linux-sgx/enclave_tf_structs.h index 575e7d34dd..fb38218f68 100644 --- a/pal/src/host/linux-sgx/enclave_tf_structs.h +++ b/pal/src/host/linux-sgx/enclave_tf_structs.h @@ -38,11 +38,11 @@ struct trusted_file { sgx_chunk_hash_t* chunk_hashes; /* array of hashes over separate file chunks */ size_t uri_len; lruc_context_t* cache; - uint64_t times_first_chunk_loaded; /* gives a hint how many times a file's content was used */ + uint64_t usage_count; /* gives a hint on how many times a file was used */ char uri[]; /* must be NULL-terminated */ }; -typedef struct _tf_chunk { +struct tf_chunk { uint64_t chunk_number; uint8_t data[TRUSTED_CHUNK_SIZE]; -} tf_chunk_t; +}; diff --git a/pal/src/host/linux-sgx/meson.build b/pal/src/host/linux-sgx/meson.build index e10defc768..a0d759975c 100644 --- a/pal/src/host/linux-sgx/meson.build +++ b/pal/src/host/linux-sgx/meson.build @@ -125,6 +125,7 @@ libpal_sgx = shared_library('pal', ], dependencies: [ + common_lru_cache_dep, common_dep, cryptoadapter_dep, ioctls_dep, diff --git a/pal/src/host/linux-sgx/pal_files.c b/pal/src/host/linux-sgx/pal_files.c index 38c794da43..ccbc7b7dff 100644 --- a/pal/src/host/linux-sgx/pal_files.c +++ b/pal/src/host/linux-sgx/pal_files.c @@ -137,8 +137,6 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, /* we lazily update the size of the trusted file */ tf->size = st.st_size; - tf->cache = lruc_create(); - tf->times_first_chunk_loaded = 0; ret = load_trusted_or_allowed_file(tf, hdl, do_create, &chunk_hashes, &total, &umem); if (ret < 0) goto fail; @@ -146,7 +144,7 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, hdl->file.chunk_hashes = chunk_hashes; hdl->file.total = total; hdl->file.umem = umem; - + hdl->file.tf = tf; *handle = hdl; return 0; @@ -187,9 +185,9 @@ static int64_t file_read(PAL_HANDLE handle, uint64_t offset, uint64_t count, voi off_t aligned_offset = ALIGN_DOWN(offset, TRUSTED_CHUNK_SIZE); off_t aligned_end = ALIGN_UP(end, TRUSTED_CHUNK_SIZE); - ret = copy_and_verify_trusted_file(handle->file.realpath, buffer, handle->file.umem, - aligned_offset, aligned_end, offset, end, chunk_hashes, - total); + ret = copy_and_verify_trusted_file(handle->file.tf, handle->file.realpath, buffer, + handle->file.umem, aligned_offset, aligned_end, offset, end, + chunk_hashes, total); if (ret < 0) return ret; @@ -227,6 +225,17 @@ static void file_destroy(PAL_HANDLE handle) { ocall_munmap_untrusted(handle->file.umem, handle->file.total); } + spinlock_lock(&g_trusted_file_lock); + if (handle->file.tf->cache) { + struct tf_chunk *chunk; + while ((chunk = lruc_get_last(handle->file.tf->cache)) != NULL) { + free(chunk); + lruc_remove_last(handle->file.tf->cache); + } + lruc_destroy(handle->file.tf->cache); + } + spinlock_unlock(&g_trusted_file_lock); + int ret = ocall_close(handle->file.fd); if (ret < 0) { log_error("closing file host fd %d failed: %s", handle->file.fd, unix_strerror(ret)); @@ -311,9 +320,9 @@ static int file_map(PAL_HANDLE handle, void* addr, pal_prot_flags_t prot, uint64 goto out; } - ret = copy_and_verify_trusted_file(handle->file.realpath, addr, handle->file.umem, - aligned_offset, aligned_end, offset, end, chunk_hashes, - handle->file.total); + ret = copy_and_verify_trusted_file(handle->file.tf, handle->file.realpath, addr, + handle->file.umem, aligned_offset, aligned_end, + offset, end, chunk_hashes, handle->file.total); if (ret < 0) { log_error("file_map - copy & verify on trusted file: %s", pal_strerror(ret)); goto out; diff --git a/pal/src/host/linux-sgx/pal_host.h b/pal/src/host/linux-sgx/pal_host.h index 1248fdef7e..b8399b1ddd 100644 --- a/pal/src/host/linux-sgx/pal_host.h +++ b/pal/src/host/linux-sgx/pal_host.h @@ -54,6 +54,7 @@ typedef struct { sgx_chunk_hash_t* chunk_hashes; /* array of hashes of file chunks */ void* umem; /* valid only when chunk_hashes != NULL */ bool seekable; /* regular files are seekable, FIFO pipes are not */ + struct trusted_file* tf; } file; struct { diff --git a/pal/src/host/linux-sgx/pal_main.c b/pal/src/host/linux-sgx/pal_main.c index ae958bb38e..bf07397374 100644 --- a/pal/src/host/linux-sgx/pal_main.c +++ b/pal/src/host/linux-sgx/pal_main.c @@ -866,7 +866,7 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void* } ret = toml_int_in(g_pal_public_state.manifest_root, "sgx.tf_max_chunks_in_cache", - /*defaultval=*/0, &g_tf_max_chunks_in_cache); + /*defaultval=*/0, &g_tf_max_chunks_in_cache); if (ret < 0) { log_error("Cannot parse 'sgx.tf_max_chunks_in_cache'"); ocall_exit(1, /*is_exitgroup=*/true); From 4e43ee04ec877440f2b1a871063372bfb628a43b Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Wed, 6 Mar 2024 12:35:25 +0530 Subject: [PATCH 08/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- pal/src/host/linux-sgx/enclave_framework.c | 27 ++++++++++++---------- pal/src/host/linux-sgx/pal_files.c | 10 ++++---- pal/src/host/linux-sgx/pal_main.c | 2 +- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index e933e1aed8..d1a38bb626 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -23,7 +23,7 @@ static int register_file(const char* uri, const char* hash_str, bool check_dupli uintptr_t g_enclave_base; uintptr_t g_enclave_top; bool g_allowed_files_warn = false; -size_t g_tf_max_chunks_in_cache = 0; +extern int64_t g_tf_max_chunks_in_cache; /* * SGX's EGETKEY(SEAL_KEY) uses three masks as key-derivation material: @@ -519,9 +519,8 @@ int load_trusted_or_allowed_file(struct trusted_file* tf, PAL_HANDLE file, bool } spinlock_lock(&g_trusted_file_lock); - if (!(tf->cache = lruc_create())) + if (g_tf_max_chunks_in_cache > 0 && !(tf->cache = lruc_create())) return -PAL_ERROR_NOMEM; - tf->usage_count = 0; if (tf->chunk_hashes) { *out_chunk_hashes = tf->chunk_hashes; @@ -631,7 +630,7 @@ static void set_file_check_policy(int policy) { static int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, uint64_t chunk_size, uint64_t chunk_number) { - if (g_tf_max_chunks_in_cache == 0) + if (g_tf_max_chunks_in_cache == 0 || !tf->cache) return 0; // Counts the number of times a file is open and reused @@ -656,7 +655,7 @@ static int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, return -PAL_ERROR_NOMEM; } - if (lruc_size(tf->cache) > g_tf_max_chunks_in_cache) { + if (lruc_size(tf->cache) > (size_t) g_tf_max_chunks_in_cache) { free(lruc_get_last(tf->cache)); lruc_remove_last(tf->cache); #ifdef DEBUG @@ -699,13 +698,15 @@ int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint for (; chunk_offset < aligned_end; chunk_offset += TRUSTED_CHUNK_SIZE, chunk_hashes_item++, chunk_number++) { size_t chunk_size = MIN(file_size - chunk_offset, TRUSTED_CHUNK_SIZE); off_t chunk_end = chunk_offset + chunk_size; - struct tf_chunk *chunk; + struct tf_chunk *chunk = NULL; - spinlock_lock(&g_trusted_file_lock); - chunk = (struct tf_chunk*)lruc_get(tf->cache, chunk_number); - spinlock_unlock(&g_trusted_file_lock); + if (g_tf_max_chunks_in_cache > 0 && tf->cache) { + spinlock_lock(&g_trusted_file_lock); + chunk = (struct tf_chunk*)lruc_get(tf->cache, chunk_number); + spinlock_unlock(&g_trusted_file_lock); + } - if (g_tf_max_chunks_in_cache > 0 && chunk != NULL) { + if (chunk != NULL) { if (chunk_offset >= offset && chunk_end <= end) { memcpy(buf_pos, chunk->data, chunk_size); @@ -749,8 +750,8 @@ int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint buf_pos += chunk_size; } else { /* if current chunk-to-copy only partially overlaps with the requested region-to-copy, - * read the file contents into a scratch buffer, verify hash and then copy only the part - * needed by the caller */ + * read the file contents into a scratch buffer, verify hash and then copy only the part + * needed by the caller */ if (!sgx_copy_to_enclave(tmp_chunk, chunk_size, umem + chunk_offset, chunk_size)) { ret = -PAL_ERROR_DENIED; goto failed; @@ -831,6 +832,8 @@ static int register_file(const char* uri, const char* hash_str, bool check_dupli new->chunk_hashes = NULL; new->allowed = false; new->uri_len = uri_len; + new->cache = NULL; + new->usage_count = 0; memcpy(new->uri, uri, uri_len + 1); if (hash_str) { diff --git a/pal/src/host/linux-sgx/pal_files.c b/pal/src/host/linux-sgx/pal_files.c index ccbc7b7dff..4bc0a298c4 100644 --- a/pal/src/host/linux-sgx/pal_files.c +++ b/pal/src/host/linux-sgx/pal_files.c @@ -24,6 +24,8 @@ #include "path_utils.h" #include "stat.h" +extern int64_t g_tf_max_chunks_in_cache; + /* this macro is used to emulate mmap() via pread() in chunks of 128MB (mmapped files may be many * GBs in size, and a pread OCALL could fail with -ENOMEM, so we cap to reasonably small size) */ #define MAX_READ_SIZE (PRESET_PAGESIZE * 1024 * 32) @@ -100,7 +102,6 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, hdl->file.fd = fd; hdl->file.seekable = !S_ISFIFO(st.st_mode); hdl->file.total = st.st_size; - *handle = hdl; return 0; } @@ -225,16 +226,17 @@ static void file_destroy(PAL_HANDLE handle) { ocall_munmap_untrusted(handle->file.umem, handle->file.total); } - spinlock_lock(&g_trusted_file_lock); - if (handle->file.tf->cache) { + if (g_tf_max_chunks_in_cache > 0 && handle->file.tf->cache) { + spinlock_lock(&g_trusted_file_lock); struct tf_chunk *chunk; while ((chunk = lruc_get_last(handle->file.tf->cache)) != NULL) { free(chunk); lruc_remove_last(handle->file.tf->cache); } lruc_destroy(handle->file.tf->cache); + handle->file.tf->cache = NULL; + spinlock_unlock(&g_trusted_file_lock); } - spinlock_unlock(&g_trusted_file_lock); int ret = ocall_close(handle->file.fd); if (ret < 0) { diff --git a/pal/src/host/linux-sgx/pal_main.c b/pal/src/host/linux-sgx/pal_main.c index bf07397374..0bbd995572 100644 --- a/pal/src/host/linux-sgx/pal_main.c +++ b/pal/src/host/linux-sgx/pal_main.c @@ -409,7 +409,7 @@ extern void* g_enclave_top; extern bool g_allowed_files_warn; extern uint64_t g_tsc_hz; extern size_t g_unused_tcs_pages_num; -extern int64_t g_tf_max_chunks_in_cache; +int64_t g_tf_max_chunks_in_cache = 0; static int print_warnings_on_insecure_configs(PAL_HANDLE parent_process) { int ret; From 874fcd7354f7c8076650d7dde05eb47ab64def4b Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Wed, 6 Mar 2024 12:44:26 +0530 Subject: [PATCH 09/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- pal/src/host/linux-sgx/enclave_tf.h | 1 + pal/src/host/linux-sgx/pal_files.c | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pal/src/host/linux-sgx/enclave_tf.h b/pal/src/host/linux-sgx/enclave_tf.h index f7410693c7..1cbaf9d0a4 100644 --- a/pal/src/host/linux-sgx/enclave_tf.h +++ b/pal/src/host/linux-sgx/enclave_tf.h @@ -61,6 +61,7 @@ int load_trusted_or_allowed_file(struct trusted_file* tf, PAL_HANDLE file, bool /*! * \brief Copy and check file contents from untrusted outside buffer to in-enclave buffer * + * \param tf Trusted file struct corresponding to this file. * \param path File path (currently only for a log message). * \param buf In-enclave buffer where contents of the file are copied. * \param umem Start of untrusted file memory mapped outside the enclave. diff --git a/pal/src/host/linux-sgx/pal_files.c b/pal/src/host/linux-sgx/pal_files.c index 4bc0a298c4..99c1902501 100644 --- a/pal/src/host/linux-sgx/pal_files.c +++ b/pal/src/host/linux-sgx/pal_files.c @@ -102,6 +102,7 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, hdl->file.fd = fd; hdl->file.seekable = !S_ISFIFO(st.st_mode); hdl->file.total = st.st_size; + *handle = hdl; return 0; } @@ -146,6 +147,7 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, hdl->file.total = total; hdl->file.umem = umem; hdl->file.tf = tf; + *handle = hdl; return 0; From 5a1f8fb6915d0d0493106317ec88711ec2622f2e Mon Sep 17 00:00:00 2001 From: jkr0103 Date: Mon, 11 Mar 2024 16:55:40 +0530 Subject: [PATCH 10/10] fixup! Page Cache support for trusted files Signed-off-by: jkr0103 --- pal/src/host/linux-sgx/enclave_framework.c | 34 ++++++++++----------- pal/src/host/linux-sgx/enclave_tf.h | 2 +- pal/src/host/linux-sgx/enclave_tf_structs.h | 9 ------ pal/src/host/linux-sgx/pal_files.c | 18 +++++------ pal/src/host/linux-sgx/pal_host.h | 9 ++++++ 5 files changed, 35 insertions(+), 37 deletions(-) diff --git a/pal/src/host/linux-sgx/enclave_framework.c b/pal/src/host/linux-sgx/enclave_framework.c index d1a38bb626..e07188db06 100644 --- a/pal/src/host/linux-sgx/enclave_framework.c +++ b/pal/src/host/linux-sgx/enclave_framework.c @@ -517,9 +517,9 @@ int load_trusted_or_allowed_file(struct trusted_file* tf, PAL_HANDLE file, bool goto fail; } } - + assert(!(file->file.cache)); spinlock_lock(&g_trusted_file_lock); - if (g_tf_max_chunks_in_cache > 0 && !(tf->cache = lruc_create())) + if (g_tf_max_chunks_in_cache > 0 && !(file->file.cache = lruc_create())) return -PAL_ERROR_NOMEM; if (tf->chunk_hashes) { @@ -628,20 +628,20 @@ static void set_file_check_policy(int policy) { g_file_check_policy = policy; } -static int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, +static int tf_append_chunk(PAL_HANDLE handle, uint8_t* chunk, uint64_t chunk_size, uint64_t chunk_number) { - if (g_tf_max_chunks_in_cache == 0 || !tf->cache) + if (g_tf_max_chunks_in_cache == 0) return 0; // Counts the number of times a file is open and reused - if (chunk_number == 0 && tf->usage_count <= 10) { + if (chunk_number == 0 && handle->file.usage_count <= 10) { spinlock_lock(&g_trusted_file_lock); - tf->usage_count++; + handle->file.usage_count++; spinlock_unlock(&g_trusted_file_lock); } // Add file chunks to cache only if the file is reused for 10 times or more - if (tf->usage_count > 10) { + if (handle->file.usage_count > 10) { struct tf_chunk* new_chunk = (struct tf_chunk*)malloc(sizeof(struct tf_chunk)); if (!new_chunk) { return -PAL_ERROR_NOMEM; @@ -650,14 +650,14 @@ static int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, memcpy(new_chunk->data, chunk, chunk_size); spinlock_lock(&g_trusted_file_lock); - if (!lruc_add(tf->cache, chunk_number, new_chunk)) { + if (!lruc_add(handle->file.cache, chunk_number, new_chunk)) { free(new_chunk); return -PAL_ERROR_NOMEM; } - if (lruc_size(tf->cache) > (size_t) g_tf_max_chunks_in_cache) { - free(lruc_get_last(tf->cache)); - lruc_remove_last(tf->cache); + if (lruc_size(handle->file.cache) > (size_t) g_tf_max_chunks_in_cache) { + free(lruc_get_last(handle->file.cache)); + lruc_remove_last(handle->file.cache); #ifdef DEBUG static int tf_cache_log_throttler = 0; if (++tf_cache_log_throttler == 100) { @@ -674,7 +674,7 @@ static int tf_append_chunk(struct trusted_file* tf, uint8_t* chunk, return 0; } -int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint8_t* buf, const void* umem, +int copy_and_verify_trusted_file(PAL_HANDLE handle, const char* path, uint8_t* buf, const void* umem, off_t aligned_offset, off_t aligned_end, off_t offset, off_t end, sgx_chunk_hash_t* chunk_hashes, size_t file_size) { int ret = 0; @@ -700,9 +700,9 @@ int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint off_t chunk_end = chunk_offset + chunk_size; struct tf_chunk *chunk = NULL; - if (g_tf_max_chunks_in_cache > 0 && tf->cache) { + if (g_tf_max_chunks_in_cache > 0) { spinlock_lock(&g_trusted_file_lock); - chunk = (struct tf_chunk*)lruc_get(tf->cache, chunk_number); + chunk = (struct tf_chunk*)lruc_get(handle->file.cache, chunk_number); spinlock_unlock(&g_trusted_file_lock); } @@ -739,7 +739,7 @@ int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint goto failed; } - ret = tf_append_chunk(tf, buf_pos, chunk_size, chunk_number); + ret = tf_append_chunk(handle, buf_pos, chunk_size, chunk_number); if (ret < 0) goto failed; @@ -757,7 +757,7 @@ int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint goto failed; } - ret = tf_append_chunk(tf, tmp_chunk, chunk_size, chunk_number); + ret = tf_append_chunk(handle, tmp_chunk, chunk_size, chunk_number); if (ret < 0) goto failed; @@ -832,8 +832,6 @@ static int register_file(const char* uri, const char* hash_str, bool check_dupli new->chunk_hashes = NULL; new->allowed = false; new->uri_len = uri_len; - new->cache = NULL; - new->usage_count = 0; memcpy(new->uri, uri, uri_len + 1); if (hash_str) { diff --git a/pal/src/host/linux-sgx/enclave_tf.h b/pal/src/host/linux-sgx/enclave_tf.h index 1cbaf9d0a4..9df73b040b 100644 --- a/pal/src/host/linux-sgx/enclave_tf.h +++ b/pal/src/host/linux-sgx/enclave_tf.h @@ -74,7 +74,7 @@ int load_trusted_or_allowed_file(struct trusted_file* tf, PAL_HANDLE file, bool * * \returns 0 on success, negative error code on failure */ -int copy_and_verify_trusted_file(struct trusted_file* tf, const char* path, uint8_t* buf, +int copy_and_verify_trusted_file(PAL_HANDLE handle, const char* path, uint8_t* buf, const void* umem, off_t aligned_offset, off_t aligned_end, off_t offset, off_t end,sgx_chunk_hash_t* chunk_hashes, size_t file_size); diff --git a/pal/src/host/linux-sgx/enclave_tf_structs.h b/pal/src/host/linux-sgx/enclave_tf_structs.h index fb38218f68..f591b38bf7 100644 --- a/pal/src/host/linux-sgx/enclave_tf_structs.h +++ b/pal/src/host/linux-sgx/enclave_tf_structs.h @@ -8,8 +8,6 @@ #include #include "list.h" -#include "lru_cache.h" -#include "pal_linux_defs.h" enum { FILE_CHECK_POLICY_STRICT = 0, @@ -37,12 +35,5 @@ struct trusted_file { sgx_file_hash_t file_hash; /* hash over the whole file, retrieved from the manifest */ sgx_chunk_hash_t* chunk_hashes; /* array of hashes over separate file chunks */ size_t uri_len; - lruc_context_t* cache; - uint64_t usage_count; /* gives a hint on how many times a file was used */ char uri[]; /* must be NULL-terminated */ }; - -struct tf_chunk { - uint64_t chunk_number; - uint8_t data[TRUSTED_CHUNK_SIZE]; -}; diff --git a/pal/src/host/linux-sgx/pal_files.c b/pal/src/host/linux-sgx/pal_files.c index 99c1902501..7ae454270d 100644 --- a/pal/src/host/linux-sgx/pal_files.c +++ b/pal/src/host/linux-sgx/pal_files.c @@ -66,6 +66,8 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, } init_handle_hdr(hdl, PAL_TYPE_FILE); + hdl->file.cache = NULL; + hdl->file.usage_count = 0; hdl->flags |= PAL_HANDLE_FD_READABLE | PAL_HANDLE_FD_WRITABLE; hdl->file.realpath = normpath; @@ -147,7 +149,6 @@ static int file_open(PAL_HANDLE* handle, const char* type, const char* uri, hdl->file.total = total; hdl->file.umem = umem; hdl->file.tf = tf; - *handle = hdl; return 0; @@ -188,7 +189,7 @@ static int64_t file_read(PAL_HANDLE handle, uint64_t offset, uint64_t count, voi off_t aligned_offset = ALIGN_DOWN(offset, TRUSTED_CHUNK_SIZE); off_t aligned_end = ALIGN_UP(end, TRUSTED_CHUNK_SIZE); - ret = copy_and_verify_trusted_file(handle->file.tf, handle->file.realpath, buffer, + ret = copy_and_verify_trusted_file(handle, handle->file.realpath, buffer, handle->file.umem, aligned_offset, aligned_end, offset, end, chunk_hashes, total); if (ret < 0) @@ -228,15 +229,14 @@ static void file_destroy(PAL_HANDLE handle) { ocall_munmap_untrusted(handle->file.umem, handle->file.total); } - if (g_tf_max_chunks_in_cache > 0 && handle->file.tf->cache) { + if (g_tf_max_chunks_in_cache > 0 && handle->file.cache) { spinlock_lock(&g_trusted_file_lock); - struct tf_chunk *chunk; - while ((chunk = lruc_get_last(handle->file.tf->cache)) != NULL) { + struct tf_chunk* chunk; + while ((chunk = lruc_get_last(handle->file.cache)) != NULL) { free(chunk); - lruc_remove_last(handle->file.tf->cache); + lruc_remove_last(handle->file.cache); } - lruc_destroy(handle->file.tf->cache); - handle->file.tf->cache = NULL; + lruc_destroy(handle->file.cache); spinlock_unlock(&g_trusted_file_lock); } @@ -324,7 +324,7 @@ static int file_map(PAL_HANDLE handle, void* addr, pal_prot_flags_t prot, uint64 goto out; } - ret = copy_and_verify_trusted_file(handle->file.tf, handle->file.realpath, addr, + ret = copy_and_verify_trusted_file(handle, handle->file.realpath, addr, handle->file.umem, aligned_offset, aligned_end, offset, end, chunk_hashes, handle->file.total); if (ret < 0) { diff --git a/pal/src/host/linux-sgx/pal_host.h b/pal/src/host/linux-sgx/pal_host.h index b8399b1ddd..5ebc7fd9be 100644 --- a/pal/src/host/linux-sgx/pal_host.h +++ b/pal/src/host/linux-sgx/pal_host.h @@ -15,6 +15,8 @@ #include #include +#include "lru_cache.h" +#include "pal_linux_defs.h" #include "enclave_tf_structs.h" #include "list.h" #include "spinlock.h" @@ -27,6 +29,11 @@ struct pal_handle_thread { void* param; }; +struct tf_chunk { + uint64_t chunk_number; + uint8_t data[TRUSTED_CHUNK_SIZE]; +}; + /* RPC streams are encrypted with 256-bit AES keys */ typedef uint8_t PAL_SESSION_KEY[32]; @@ -55,6 +62,8 @@ typedef struct { void* umem; /* valid only when chunk_hashes != NULL */ bool seekable; /* regular files are seekable, FIFO pipes are not */ struct trusted_file* tf; + lruc_context_t* cache; + uint64_t usage_count; /* gives a hint on how many times a file was used */ } file; struct {