From a6a84f726ef478387a09dbc5155c1332e62c7eb5 Mon Sep 17 00:00:00 2001 From: Li Lei Date: Tue, 24 Jul 2018 19:13:15 -0700 Subject: [PATCH 1/8] EDMM: Dynamic Heap Management --- Pal/src/host/Linux-SGX/enclave_entry.S | 46 ++++++++++++++++++++++ Pal/src/host/Linux-SGX/enclave_framework.c | 30 ++++++++++++++ Pal/src/host/Linux-SGX/enclave_pages.c | 15 +++++++ Pal/src/host/Linux-SGX/pal_security.h | 3 ++ Pal/src/host/Linux-SGX/sgx_arch.h | 8 ++++ Pal/src/host/Linux-SGX/sgx_framework.c | 2 +- Pal/src/host/Linux-SGX/sgx_main.c | 21 ++++++++-- Pal/src/host/Linux-SGX/signer/pal-sgx-sign | 23 ++++++++++- 8 files changed, 141 insertions(+), 7 deletions(-) diff --git a/Pal/src/host/Linux-SGX/enclave_entry.S b/Pal/src/host/Linux-SGX/enclave_entry.S index 0d4b25b6ec..afcd990fbd 100644 --- a/Pal/src/host/Linux-SGX/enclave_entry.S +++ b/Pal/src/host/Linux-SGX/enclave_entry.S @@ -353,6 +353,52 @@ sgx_getkey: .cfi_endproc .size sgx_getkey, .-sgx_getkey +/* + * sgx_accept: + * EACCEPT pages for dynamic memory management + */ + .global sgx_do_eaccept + .type sgx_do_eaccept, @function +sgx_do_eaccept: + .cfi_startproc + + push %rbx + push %rcx + mov %rdi, %rbx + mov %rsi, %rcx + mov $EACCEPT, %rax + ENCLU + pop %rcx + pop %rbx + ret + + .cfi_endproc + .size sgx_do_eaccept, .-sgx_do_eaccept + +/* + * sgx_emodpe: + * EMODPE pages for dynamic memory management + */ + + .global sgx_do_emodpe + .type sgx_do_emodpe, @function +sgx_do_emodpe: + .cfi_startproc + + push %rbx + push %rcx + mov %rdi, %rbx + mov %rsi, %rcx + mov $EMODPE, %rax + ENCLU + pop %rcx + pop %rbx + ret + + .cfi_endproc + .size sgx_do_emodpe, .-sgx_do_emodpe + + /* * rdrand: * Get hardware generated random value. diff --git a/Pal/src/host/Linux-SGX/enclave_framework.c b/Pal/src/host/Linux-SGX/enclave_framework.c index 618efb8e68..dbba91ba59 100644 --- a/Pal/src/host/Linux-SGX/enclave_framework.c +++ b/Pal/src/host/Linux-SGX/enclave_framework.c @@ -97,6 +97,36 @@ int sgx_verify_report (sgx_arch_report_t * report) return 0; } + +#define SE_DECLSPEC_ALIGN(x) __attribute__((aligned(x))) + +int sgx_accept_pages(uint64_t sfl, size_t lo, size_t hi) +{ + size_t addr = hi; + SE_DECLSPEC_ALIGN(sizeof(sgx_arch_secinfo_t)) sgx_arch_secinfo_t si; + si.flags = sfl; + for (uint16_t i = 0; i < (sizeof(si.reserved)/sizeof(si.reserved[0])); i++) + si.reserved[i] = 0; + SGX_DBG(DBG_E, "sgx_accept_pages: %p - %p\n", lo, hi); + SE_DECLSPEC_ALIGN(sizeof(sgx_arch_secinfo_t)) sgx_arch_secinfo_t smi = si; + smi.flags |= SGX_SECINFO_FLAGS_X; + + while (lo < addr) + { + addr -= PRESET_PAGESIZE; + int rc = sgx_do_eaccept(&si, addr); + + /* FIXME: Need a better handle here, adding the flow for checking multiple EACCEPT on the same page */ + if (rc != 0) { +// SGX_DBG(DBG_E, "eaccept fails: %d\n", rc); +// return rc; + continue; + } + rc = sgx_do_emodpe(&smi, addr); + } + return 0; +} + int init_enclave_key (void) { sgx_arch_keyrequest_t keyrequest; diff --git a/Pal/src/host/Linux-SGX/enclave_pages.c b/Pal/src/host/Linux-SGX/enclave_pages.c index 75f33b4a12..73e49f4b55 100644 --- a/Pal/src/host/Linux-SGX/enclave_pages.c +++ b/Pal/src/host/Linux-SGX/enclave_pages.c @@ -70,6 +70,17 @@ static void assert_vma_list (void) #endif } +void allocate_page_range(void * addr, uint64_t size) +{ + uint64_t start_addr = addr; + uint64_t end_addr = addr + size; + struct heap_vma * vma; + uint64_t accept_flags = SGX_SECINFO_FLAGS_R | SGX_SECINFO_FLAGS_W | + SGX_SECINFO_FLAGS_REG | SGX_SECINFO_FLAGS_PENDING; + + sgx_accept_pages(accept_flags, start_addr, end_addr); +} + // TODO: This function should be fixed to always either return exactly `addr` or // fail. void * get_reserved_pages(void * addr, uint64_t size) @@ -158,6 +169,10 @@ void * get_reserved_pages(void * addr, uint64_t size) listp_first_entry(&heap_vma_list, struct heap_vma, list); } + /* Dynamically Request EPC pages in EDMM Mode */ + if (pal_sec.edmm_mode) + allocate_page_range (addr, size); + if (prev && next) SGX_DBG(DBG_M, "insert vma between %p-%p and %p-%p\n", next->bottom, next->top, prev->bottom, prev->top); diff --git a/Pal/src/host/Linux-SGX/pal_security.h b/Pal/src/host/Linux-SGX/pal_security.h index 6dd5195c7c..0dfbf2cfdb 100644 --- a/Pal/src/host/Linux-SGX/pal_security.h +++ b/Pal/src/host/Linux-SGX/pal_security.h @@ -39,6 +39,9 @@ struct pal_sec { sgx_arch_hash_t mrsigner; sgx_arch_attributes_t enclave_attributes; + /* edmm mode */ + uint8_t edmm_mode; + /* remaining heap usable by application */ PAL_PTR heap_min, heap_max; diff --git a/Pal/src/host/Linux-SGX/sgx_arch.h b/Pal/src/host/Linux-SGX/sgx_arch.h index c6a8f23719..ab739ad48f 100644 --- a/Pal/src/host/Linux-SGX/sgx_arch.h +++ b/Pal/src/host/Linux-SGX/sgx_arch.h @@ -166,6 +166,11 @@ typedef struct { #define SGX_SECINFO_FLAGS_TCS 0x100 #define SGX_SECINFO_FLAGS_REG 0x200 +/* EDMM PAGE STATUS */ +#define SGX_SECINFO_FLAGS_PENDING 0x08 +#define SGX_SECINFO_FLAGS_MODIFIED 0x010 +#define SGX_SECINFO_FLAGS_PR 0x020 + typedef struct { /* header part (signed) */ uint32_t header[4], vendor; @@ -264,6 +269,9 @@ typedef uint8_t sgx_arch_key128_t[16] __attribute__((aligned(16))); #define EREPORT 0 #define EGETKEY 1 #define EEXIT 4 +#define EACCEPT 5 +#define EMODPE 6 + #define LAUNCH_KEY 0 #define PROVISION_KEY 1 diff --git a/Pal/src/host/Linux-SGX/sgx_framework.c b/Pal/src/host/Linux-SGX/sgx_framework.c index c4ea9f0e92..18a8faf8a0 100644 --- a/Pal/src/host/Linux-SGX/sgx_framework.c +++ b/Pal/src/host/Linux-SGX/sgx_framework.c @@ -177,7 +177,7 @@ int create_enclave(sgx_arch_secs_t * secs, #endif if (IS_ERR(ret)) { - SGX_DBG(DBG_I, "enclave ECREATE failed in enclave creation ioctl - %d\n", ERRNO(ret)); + SGX_DBG(DBG_E, "enclave ECREATE failed in enclave creation ioctl - %d\n", ERRNO(ret)); return -ERRNO(ret); } diff --git a/Pal/src/host/Linux-SGX/sgx_main.c b/Pal/src/host/Linux-SGX/sgx_main.c index eeefee378a..4eb8c8116e 100644 --- a/Pal/src/host/Linux-SGX/sgx_main.c +++ b/Pal/src/host/Linux-SGX/sgx_main.c @@ -288,6 +288,17 @@ int initialize_enclave (struct pal_enclave * enclave) goto err; } + /* Reading sgx.edmm_mode from manifest */ + if (get_config(enclave->config, "sgx.edmm_mode", cfgbuf, CONFIG_MAX) <= 0) { + SGX_DBG(DBG_E, "edmm_mode is not specified, edmm mode is disabled by default\n"); + enclave->pal_sec.edmm_mode = 0; + } + + enclave->pal_sec.edmm_mode = parse_int(cfgbuf); + + if (enclave->pal_sec.edmm_mode) + enclave->size = parse_int("4G"); + /* Reading sgx.static_address from manifest */ if (get_config(enclave->config, "sgx.static_address", cfgbuf, CONFIG_MAX) > 0 && cfgbuf[0] == '1') @@ -465,10 +476,12 @@ int initialize_enclave (struct pal_enclave * enclave) areas[i].fd, 0); add_pages: - TRY(add_pages_to_enclave, - &enclave_secs, (void *) areas[i].addr, data, areas[i].size, - areas[i].type, areas[i].prot, areas[i].skip_eextend, - areas[i].desc); + if (!enclave->pal_sec.edmm_mode || (!strcmp_static(areas[i].desc, "free"))) { + TRY(add_pages_to_enclave, + &enclave_secs, (void *) areas[i].addr, data, areas[i].size, + areas[i].type, areas[i].prot, areas[i].skip_eextend, + areas[i].desc); + } if (data) INLINE_SYSCALL(munmap, 2, data, areas[i].size); diff --git a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign index c5301d054e..529ae47ef3 100755 --- a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign +++ b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign @@ -176,6 +176,10 @@ def get_enclave_attributes(manifest): if manifest_options[opt] in attributes: attributes.pop(manifest_options[opt]) + if 'sgx.disable_avx' in manifest: + if manifest['sgx.disable_avx'] == '1': + attributes.remove('XFRM_AVX') + flags_raw = struct.pack(">sys.stderr, "Attributes:" print >>sys.stderr, " size: %d" % (attr['enclave_size']) print >>sys.stderr, " threadnum: %d" % (attr['thread_num']) @@ -714,8 +732,9 @@ if __name__ == "__main__": print >>sys.stderr, " flags: %016x" % (bytes_to_int(attr['flags'])) print >>sys.stderr, " xfrms: %016x" % (bytes_to_int(attr['xfrms'])) print >>sys.stderr, " miscs: %08x" % (bytes_to_int(attr['miscs'])) + print >>sys.stderr, " edmm_mode: %d" % (attr['edmm_mode']) - # Get trusted checksums and measurements + # Get trusted checksums and measurements print >>sys.stderr, "Trusted files:" for key, val in get_trusted_files(manifest, args).items(): (uri, target, checksum) = val From 08c3dc27b75574343028b7d43b31bd9060a19972 Mon Sep 17 00:00:00 2001 From: Li Lei Date: Fri, 27 Jul 2018 01:44:23 -0700 Subject: [PATCH 2/8] EDMM: Dynamic Stack Management Support --- Pal/src/host/Linux-SGX/ecall_types.h | 1 + Pal/src/host/Linux-SGX/enclave_ecalls.c | 40 ++++++++++- Pal/src/host/Linux-SGX/enclave_entry.S | 80 +++++++++++++++++----- Pal/src/host/Linux-SGX/enclave_framework.c | 10 +-- Pal/src/host/Linux-SGX/enclave_pages.c | 7 +- Pal/src/host/Linux-SGX/sgx_api.h | 5 ++ Pal/src/host/Linux-SGX/sgx_enclave.c | 6 ++ Pal/src/host/Linux-SGX/sgx_enclave.h | 2 + Pal/src/host/Linux-SGX/sgx_entry.S | 2 + Pal/src/host/Linux-SGX/sgx_exception.c | 17 ++++- Pal/src/host/Linux-SGX/sgx_internal.h | 7 ++ Pal/src/host/Linux-SGX/sgx_main.c | 24 ++++++- Pal/src/host/Linux-SGX/sgx_tls.h | 10 ++- Pal/src/host/Linux-SGX/signer/pal-sgx-sign | 6 +- 14 files changed, 184 insertions(+), 33 deletions(-) diff --git a/Pal/src/host/Linux-SGX/ecall_types.h b/Pal/src/host/Linux-SGX/ecall_types.h index 437cd98255..3f13431df2 100644 --- a/Pal/src/host/Linux-SGX/ecall_types.h +++ b/Pal/src/host/Linux-SGX/ecall_types.h @@ -4,6 +4,7 @@ enum { ECALL_ENCLAVE_START = 0, ECALL_THREAD_START, + ECALL_STACK_EXPAND, ECALL_NR, }; diff --git a/Pal/src/host/Linux-SGX/enclave_ecalls.c b/Pal/src/host/Linux-SGX/enclave_ecalls.c index 82bacd7749..d0c1e0401a 100644 --- a/Pal/src/host/Linux-SGX/enclave_ecalls.c +++ b/Pal/src/host/Linux-SGX/enclave_ecalls.c @@ -17,6 +17,32 @@ void pal_start_thread (void); extern void * enclave_base, * enclave_top; +void pal_expand_stack(unsigned long fault_addr) +{ + unsigned long stack_commit_top = GET_ENCLAVE_TLS(stack_commit_top); + unsigned long accept_flags = SGX_SECINFO_FLAGS_R | SGX_SECINFO_FLAGS_W | + SGX_SECINFO_FLAGS_REG | SGX_SECINFO_FLAGS_PENDING; + unsigned long stack_init_addr = GET_ENCLAVE_TLS(initial_stack_offset); + unsigned long end_addr = fault_addr - PRESET_PAGESIZE; + + SGX_DBG(DBG_E, "fault_addr, stack_commit_top, stack_init_addr: %p, %p, %p\n", + fault_addr, stack_commit_top, stack_init_addr); + if (fault_addr < (stack_init_addr - ENCLAVE_STACK_SIZE * PRESET_PAGESIZE)) { + SGX_DBG(DBG_E, "stack overrun, stop!\n"); + return ; + } + /* Bridge the gap between fault addr and top if any */ + sgx_accept_pages(accept_flags, fault_addr, stack_commit_top, 0); + + stack_commit_top = fault_addr; + /* Overgrow one more page */ + if (end_addr >= stack_init_addr - ENCLAVE_STACK_SIZE * PRESET_PAGESIZE) { + sgx_accept_pages(accept_flags, end_addr, fault_addr, 0); + stack_commit_top = fault_addr; + } + +} + int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, void * untrusted_stack, void * enclave_base_addr) { @@ -37,7 +63,9 @@ int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, SET_ENCLAVE_TLS(exit_target, exit_target); SET_ENCLAVE_TLS(ustack_top, untrusted_stack); SET_ENCLAVE_TLS(ustack, untrusted_stack); - + SET_ENCLAVE_TLS(ocall_pending, 0); + void * utr_stack = GET_ENCLAVE_TLS(ustack); + SGX_DBG(DBG_E, "utrusted stack: %p\n", utr_stack); switch(ecall_index) { case ECALL_ENCLAVE_START: { ms_ecall_enclave_start_t * ms = @@ -47,14 +75,20 @@ int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, pal_linux_main(ms->ms_arguments, ms->ms_environments, ms->ms_sec_info); + ocall_exit(); break; } case ECALL_THREAD_START: pal_start_thread(); + ocall_exit(); break; + case ECALL_STACK_EXPAND: + pal_expand_stack(ecall_args); + break; } - - ocall_exit(); + utr_stack = GET_ENCLAVE_TLS(ustack); + SGX_DBG(DBG_E, "untrusted stack after: %p\n", utr_stack); + SGX_DBG(DBG_E, "exit_target: %p\n", GET_ENCLAVE_TLS(exit_target)); return 0; } diff --git a/Pal/src/host/Linux-SGX/enclave_entry.S b/Pal/src/host/Linux-SGX/enclave_entry.S index afcd990fbd..24a251310e 100644 --- a/Pal/src/host/Linux-SGX/enclave_entry.S +++ b/Pal/src/host/Linux-SGX/enclave_entry.S @@ -12,11 +12,24 @@ enclave_entry: # RCX is the address of AEP. Other registers are not trusted. # current SSA is in RAX (Trusted) - cmp $0, %rax - jne .Lhandle_resume + # cmp $0, %rax + # jne .Lhandle_resume + + # current SSA is in RAX (Trusted) + cmp $0, %rax + je .Lnormal_enter + + # Exception for growing stacks + cmp $2, %rdi + je .Lnormal_enter + + # OCALL made in Exception context + cmp $1, %gs:SGX_OCALL_PENDING + jne .Lhandle_resume # TCS is in RBX (Trusted) +.Lnormal_enter: # AEP address in RCX (Trusted) mov %rcx, %gs:SGX_AEP @@ -36,9 +49,12 @@ enclave_entry: # to deceive the trusted PAL. # A safe design: check if %gs:SGX_EXIT_TARGET is ever assigned - mov %gs:SGX_EXIT_TARGET, %rcx - cmp $0, %rcx - jne .Lreturn_from_ocall +# mov %gs:SGX_EXIT_TARGET, %rcx +# cmp $0, %rcx +# jne .Lreturn_from_ocall + + cmp $0, %gs:SGX_OCALL_PENDING + jne .Lreturn_from_ocall # PAL convention: # RDI - index in ecall_table @@ -46,6 +62,7 @@ enclave_entry: # RDX - exit target # RCX (former RSP) - The unstrusted stack # R8 - enclave base + # R9 - address of normal return # calculate enclave base = RBX (trusted) - %gs:SGX_TCS_OFFSET sub %gs:SGX_TCS_OFFSET, %rbx @@ -54,10 +71,15 @@ enclave_entry: # push untructed stack address to RCX mov %rsp, %rcx + # handling stack grow, using exception stack + cmp $2, %rdi + je .Lhandle_exception_stack + # switch to enclve stack: enclave base + %gs:SGX_INITIAL_STACK_OFFSET add %gs:SGX_INITIAL_STACK_OFFSET, %rbx mov %rbx, %rsp +.Ldo_handle_ecall: # clear the rest of register states xor %rax, %rax xor %rbx, %rbx @@ -72,12 +94,23 @@ enclave_entry: # register states need to be carefully checked, so we move the handling # to handle_ecall() in enclave_ecalls.c call handle_ecall - + # never return to this point (should die) xor %rdi, %rdi xor %rsi, %rsi jmp .Leexit +.Lhandle_exception_stack: + add %gs:SGX_EXCEPTION_STACK_OFFSET, %rbx + mov %rbx, %rsp + + # In exception handler, ecall returns to the next + # RCX contains the return address, however occupied + # Using SGX_AEP instead + mov %gs:SGX_AEP, %r9 + mov %r9, %gs:SGX_ECALL_RET_TARGET + jmp .Ldo_handle_ecall + .Lhandle_resume: # PAL convention: # RDI - external event @@ -233,7 +266,8 @@ sgx_ocall: push %rbp mov %rsp, %gs:SGX_STACK - + + movq $1, %gs:SGX_OCALL_PENDING jmp .Leexit .Lexception_handler: @@ -251,13 +285,22 @@ sgx_ocall: xor %rbp, %rbp mov %gs:SGX_USTACK, %rsp - and $STACK_ALIGN, %rsp +# and $STACK_ALIGN, %rsp + cmp $1, %gs:SGX_OCALL_PENDING + jne .Lecall_return_setup + mov %gs:SGX_EXIT_TARGET, %rbx mov %gs:SGX_AEP, %rcx +.Lexecute_exit: mov $EEXIT, %rax ENCLU +.Lecall_return_setup: + mov %gs:SGX_ECALL_RET_TARGET, %rbx + mov %gs:SGX_AEP, %rcx + jmp .Lexecute_exit + .Lreturn_from_ocall: # PAL convention: # RDI - return value @@ -265,6 +308,9 @@ sgx_ocall: mov %rdi, %rax + # Clear ocall_pending + movq $0, %gs:SGX_OCALL_PENDING + # restore FSBASE if necessary mov %gs:SGX_FSBASE, %rbx cmp $0, %rbx @@ -357,9 +403,9 @@ sgx_getkey: * sgx_accept: * EACCEPT pages for dynamic memory management */ - .global sgx_do_eaccept - .type sgx_do_eaccept, @function -sgx_do_eaccept: + .global sgx_accept + .type sgx_accept, @function +sgx_accept: .cfi_startproc push %rbx @@ -373,16 +419,16 @@ sgx_do_eaccept: ret .cfi_endproc - .size sgx_do_eaccept, .-sgx_do_eaccept + .size sgx_accept, .-sgx_accept /* - * sgx_emodpe: + * sgx_modpe: * EMODPE pages for dynamic memory management */ - .global sgx_do_emodpe - .type sgx_do_emodpe, @function -sgx_do_emodpe: + .global sgx_modpe + .type sgx_modpe, @function +sgx_modpe: .cfi_startproc push %rbx @@ -396,7 +442,7 @@ sgx_do_emodpe: ret .cfi_endproc - .size sgx_do_emodpe, .-sgx_do_emodpe + .size sgx_modpe, .-sgx_modpe /* diff --git a/Pal/src/host/Linux-SGX/enclave_framework.c b/Pal/src/host/Linux-SGX/enclave_framework.c index dbba91ba59..b8d27c4e7b 100644 --- a/Pal/src/host/Linux-SGX/enclave_framework.c +++ b/Pal/src/host/Linux-SGX/enclave_framework.c @@ -100,21 +100,21 @@ int sgx_verify_report (sgx_arch_report_t * report) #define SE_DECLSPEC_ALIGN(x) __attribute__((aligned(x))) -int sgx_accept_pages(uint64_t sfl, size_t lo, size_t hi) +int sgx_accept_pages(uint64_t sfl, size_t lo, size_t hi, bool executable) { size_t addr = hi; SE_DECLSPEC_ALIGN(sizeof(sgx_arch_secinfo_t)) sgx_arch_secinfo_t si; si.flags = sfl; for (uint16_t i = 0; i < (sizeof(si.reserved)/sizeof(si.reserved[0])); i++) si.reserved[i] = 0; - SGX_DBG(DBG_E, "sgx_accept_pages: %p - %p\n", lo, hi); + SGX_DBG(DBG_M, "sgx_accept_pages: %p - %p, executable: %d \n", lo, hi, executable); SE_DECLSPEC_ALIGN(sizeof(sgx_arch_secinfo_t)) sgx_arch_secinfo_t smi = si; smi.flags |= SGX_SECINFO_FLAGS_X; while (lo < addr) { addr -= PRESET_PAGESIZE; - int rc = sgx_do_eaccept(&si, addr); + int rc = sgx_accept(&si, (const void *)addr); /* FIXME: Need a better handle here, adding the flow for checking multiple EACCEPT on the same page */ if (rc != 0) { @@ -122,7 +122,9 @@ int sgx_accept_pages(uint64_t sfl, size_t lo, size_t hi) // return rc; continue; } - rc = sgx_do_emodpe(&smi, addr); + if (executable){ + rc = sgx_modpe(&smi, (const void *)addr); + } } return 0; } diff --git a/Pal/src/host/Linux-SGX/enclave_pages.c b/Pal/src/host/Linux-SGX/enclave_pages.c index 73e49f4b55..0726581141 100644 --- a/Pal/src/host/Linux-SGX/enclave_pages.c +++ b/Pal/src/host/Linux-SGX/enclave_pages.c @@ -72,13 +72,12 @@ static void assert_vma_list (void) void allocate_page_range(void * addr, uint64_t size) { - uint64_t start_addr = addr; - uint64_t end_addr = addr + size; - struct heap_vma * vma; + uint64_t start_addr = (uint64_t)addr; + uint64_t end_addr = (uint64_t)addr + size; uint64_t accept_flags = SGX_SECINFO_FLAGS_R | SGX_SECINFO_FLAGS_W | SGX_SECINFO_FLAGS_REG | SGX_SECINFO_FLAGS_PENDING; - sgx_accept_pages(accept_flags, start_addr, end_addr); + sgx_accept_pages(accept_flags, start_addr, end_addr, 1); } // TODO: This function should be fixed to always either return exactly `addr` or diff --git a/Pal/src/host/Linux-SGX/sgx_api.h b/Pal/src/host/Linux-SGX/sgx_api.h index 502c7f0bbc..6bf43ed603 100644 --- a/Pal/src/host/Linux-SGX/sgx_api.h +++ b/Pal/src/host/Linux-SGX/sgx_api.h @@ -41,8 +41,13 @@ int sgx_get_report (sgx_arch_hash_t * mrenclave, int sgx_verify_report (sgx_arch_report_t * report); +int sgx_accept(sgx_arch_secinfo_t* si, const void * addr); + +int sgx_modpe(sgx_arch_secinfo_t* si, const void * addr); + uint32_t rdrand (void); uint64_t rdfsbase (void); void wrfsbase (uint64_t addr); +int sgx_accept_pages(uint64_t sfl, size_t lo, size_t hi, bool executable); #endif /* SGX_API_H */ diff --git a/Pal/src/host/Linux-SGX/sgx_enclave.c b/Pal/src/host/Linux-SGX/sgx_enclave.c index 98baa20a75..c8d2febea1 100644 --- a/Pal/src/host/Linux-SGX/sgx_enclave.c +++ b/Pal/src/host/Linux-SGX/sgx_enclave.c @@ -720,6 +720,12 @@ int ecall_thread_start (void) return sgx_ecall(ECALL_THREAD_START, NULL); } +int ecall_stack_expand(void * addr) +{ + EDEBUG(ECALL_STACK_EXPAND, addr); + return sgx_ecall(ECALL_STACK_EXPAND, addr); +} + void __abort(void) { INLINE_SYSCALL(exit_group, 1, -1); } diff --git a/Pal/src/host/Linux-SGX/sgx_enclave.h b/Pal/src/host/Linux-SGX/sgx_enclave.h index 98b54bba43..3a05149c80 100644 --- a/Pal/src/host/Linux-SGX/sgx_enclave.h +++ b/Pal/src/host/Linux-SGX/sgx_enclave.h @@ -7,3 +7,5 @@ int ecall_enclave_start (const char ** arguments, const char ** environments); int ecall_thread_start (void); + +int ecall_stack_expand (void * fault_addr); diff --git a/Pal/src/host/Linux-SGX/sgx_entry.S b/Pal/src/host/Linux-SGX/sgx_entry.S index ac4f6a92bd..84e4427298 100644 --- a/Pal/src/host/Linux-SGX/sgx_entry.S +++ b/Pal/src/host/Linux-SGX/sgx_entry.S @@ -23,6 +23,8 @@ sgx_ecall: mov $EENTER, %rax ENCLU + retq + .global async_exit_pointer .type async_exit_pointer, @function diff --git a/Pal/src/host/Linux-SGX/sgx_exception.c b/Pal/src/host/Linux-SGX/sgx_exception.c index 6fa4feaecd..d3ae5d772e 100644 --- a/Pal/src/host/Linux-SGX/sgx_exception.c +++ b/Pal/src/host/Linux-SGX/sgx_exception.c @@ -226,7 +226,8 @@ static void _DkResumeSighandler (int signum, siginfo_t * info, struct ucontext * uc) { unsigned long rip = uc->uc_mcontext.gregs[REG_RIP]; - + unsigned long rax = uc->uc_mcontext.gregs[REG_RAX]; + #if SGX_HAS_FSGSBASE == 0 if (rip != (unsigned long) async_exit_pointer && rip != (unsigned long) double_async_exit) { @@ -264,7 +265,19 @@ static void _DkResumeSighandler (int signum, siginfo_t * info, break; } #if SGX_HAS_FSGSBASE != 0 - sgx_raise(event); + unsigned long fault_addr = info->si_addr; + unsigned long stack_start_addr = current_enclave->stackinfo.start_addr; + unsigned long stack_end_addr = current_enclave->stackinfo.end_addr; + // fault happened at stack area + if (current_enclave->pal_sec.edmm_mode + && signum == SIGBUS && rax == ERESUME + && (fault_addr <= stack_start_addr + && fault_addr >= stack_end_addr)){ + ecall_stack_expand(fault_addr); + printf("return from stack expand!\n"); + } + else + sgx_raise(event); #else uc->uc_mcontext.gregs[REG_R9] = event; #endif diff --git a/Pal/src/host/Linux-SGX/sgx_internal.h b/Pal/src/host/Linux-SGX/sgx_internal.h index 1319c10b97..df99be9171 100644 --- a/Pal/src/host/Linux-SGX/sgx_internal.h +++ b/Pal/src/host/Linux-SGX/sgx_internal.h @@ -62,6 +62,11 @@ uint16_t htons (uint16_t shortval); uint32_t ntohl (uint32_t longval); uint16_t ntohs (uint16_t shortval); +struct pal_enclave_stack_info { + unsigned long start_addr; + unsigned long end_addr; +}; + struct pal_enclave { /* attributes */ unsigned long baseaddr; @@ -69,6 +74,8 @@ struct pal_enclave { unsigned long thread_num; unsigned long ssaframesize; + /* stack information */ + struct pal_enclave_stack_info stackinfo; /* files */ int manifest; int exec; diff --git a/Pal/src/host/Linux-SGX/sgx_main.c b/Pal/src/host/Linux-SGX/sgx_main.c index 4eb8c8116e..93271aef97 100644 --- a/Pal/src/host/Linux-SGX/sgx_main.c +++ b/Pal/src/host/Linux-SGX/sgx_main.c @@ -367,6 +367,12 @@ int initialize_enclave (struct pal_enclave * enclave) for (int t = 0 ; t < enclave->thread_num ; t++) set_area("stack", true, false, -1, 0, ENCLAVE_STACK_SIZE, PROT_READ|PROT_WRITE, SGX_PAGE_REG); + /* XXX: EDMM one page for exception handler stack */ + struct mem_area * exception_stack_area = NULL; + if (enclave->pal_sec.edmm_mode) + exception_stack_area = + set_area("exception_stack", true, false, -1, 0, pagesize, + PROT_READ|PROT_WRITE, SGX_PAGE_REG); struct mem_area * pal_area = set_area("pal", false, true, enclave_image, 0, 0, 0, SGX_PAGE_REG); @@ -416,6 +422,10 @@ int initialize_enclave (struct pal_enclave * enclave) PROT_READ|PROT_WRITE|PROT_EXEC, SGX_PAGE_REG); } + /* store stack range info for dynamic stack grow */ + enclave->stackinfo.start_addr = stack_areas[0].addr + ENCLAVE_STACK_SIZE * PRESET_PAGESIZE; + enclave->stackinfo.end_addr = stack_areas[enclave->thread_num - 1].addr; + for (int i = 0 ; i < area_num ; i++) { if (areas[i].fd != -1 && areas[i].is_binary) { TRY(load_enclave_binary, @@ -441,6 +451,17 @@ int initialize_enclave (struct pal_enclave * enclave) enclave_secs.baseaddr; gs->gpr = gs->ssa + enclave->ssaframesize - sizeof(sgx_arch_gpr_t); + + gs->ocall_pending = 0; + + if (enclave->pal_sec.edmm_mode){ + /* TODO: each thread should have their own exception stack */ + gs->exception_stack_offset = exception_stack_area->addr + pagesize; + gs->stack_commit_top = gs->initial_stack_offset; + //gs->thread_manage_stack_offset = thread_management_stack_area->addr + pagesize; + //printf("gs->exception_stack_offset: 0x%x\n", gs->exception_stack_offset); + } + } goto add_pages; @@ -476,7 +497,8 @@ int initialize_enclave (struct pal_enclave * enclave) areas[i].fd, 0); add_pages: - if (!enclave->pal_sec.edmm_mode || (!strcmp_static(areas[i].desc, "free"))) { + if (!enclave->pal_sec.edmm_mode || (!strcmp_static(areas[i].desc, "free") && + !strcmp_static(areas[i].desc, "stack"))) { TRY(add_pages_to_enclave, &enclave_secs, (void *) areas[i].addr, data, areas[i].size, areas[i].type, areas[i].prot, areas[i].skip_eextend, diff --git a/Pal/src/host/Linux-SGX/sgx_tls.h b/Pal/src/host/Linux-SGX/sgx_tls.h index 5f49a71a4c..149b341758 100644 --- a/Pal/src/host/Linux-SGX/sgx_tls.h +++ b/Pal/src/host/Linux-SGX/sgx_tls.h @@ -19,6 +19,11 @@ struct enclave_tls { void * ustack_top; void * ustack; void * thread; + uint64_t exception_stack_offset; + uint64_t stack_commit_top; + uint64_t ecall_ret_target; + uint64_t thread_manage_stack_offset; + uint64_t ocall_pending; }; #ifndef DEBUG @@ -56,7 +61,10 @@ extern uint64_t dummy_debug_variable; #define SGX_USTACK_TOP 0x48 #define SGX_USTACK 0x50 #define SGX_THREAD 0x58 - +#define SGX_EXCEPTION_STACK_OFFSET 0x60 +#define SGX_ECALL_RET_TARGET 0x70 +#define SGX_THREAD_MANAGE_STACK_OFFSET 0x78 +#define SGX_OCALL_PENDING 0x80 #endif #endif /* __SGX_TLS_H__ */ diff --git a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign index 529ae47ef3..5b5c0181a9 100755 --- a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign +++ b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign @@ -348,6 +348,10 @@ def get_memory_areas(manifest, attr, args): areas.append(MemoryArea('stack', size=ENCLAVE_STACK_SIZE, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) + if edmm_mode: + areas.append(MemoryArea('exception_stack', size=PAGESIZE*1, + flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) + areas.append(MemoryArea('pal', file=args['libpal'], flags=PAGEINFO_REG)) if 'exec' in args: @@ -518,7 +522,7 @@ def generate_measurement(attr, areas): do_eadd(mrenclave, a, area.flags) else: for a in range(area.addr, area.addr + area.size, PAGESIZE): - if area.desc != "free": # and area.desc != "stack": + if area.desc != "free" and area.desc != "stack": do_eadd(mrenclave, a, area.flags) print_area(area.addr, area.size, area.flags, area.desc, False) From 3cc4cb50a0d985b3cff822f12417fb05c07e9a63 Mon Sep 17 00:00:00 2001 From: Li Lei Date: Wed, 1 Aug 2018 00:31:51 -0700 Subject: [PATCH 3/8] EDMM: Dynamic Thread Creation --- Pal/src/host/Linux-SGX/ecall_types.h | 2 + Pal/src/host/Linux-SGX/enclave_ecalls.c | 96 ++++++++++++++++++++-- Pal/src/host/Linux-SGX/enclave_entry.S | 38 +++++---- Pal/src/host/Linux-SGX/enclave_framework.c | 14 ++-- Pal/src/host/Linux-SGX/pal_linux.h | 2 + Pal/src/host/Linux-SGX/sgx_enclave.c | 12 +++ Pal/src/host/Linux-SGX/sgx_enclave.h | 4 + Pal/src/host/Linux-SGX/sgx_entry.S | 14 ++++ Pal/src/host/Linux-SGX/sgx_exception.c | 14 ++-- Pal/src/host/Linux-SGX/sgx_framework.c | 22 +++++ Pal/src/host/Linux-SGX/sgx_internal.h | 5 +- Pal/src/host/Linux-SGX/sgx_main.c | 74 +++++++++++------ Pal/src/host/Linux-SGX/sgx_thread.c | 79 ++++++++++++++++-- Pal/src/host/Linux-SGX/sgx_tls.h | 10 +-- Pal/src/host/Linux-SGX/signer/pal-sgx-sign | 35 +++++--- 15 files changed, 331 insertions(+), 90 deletions(-) diff --git a/Pal/src/host/Linux-SGX/ecall_types.h b/Pal/src/host/Linux-SGX/ecall_types.h index 3f13431df2..f8c1dc726c 100644 --- a/Pal/src/host/Linux-SGX/ecall_types.h +++ b/Pal/src/host/Linux-SGX/ecall_types.h @@ -5,6 +5,8 @@ enum { ECALL_ENCLAVE_START = 0, ECALL_THREAD_START, ECALL_STACK_EXPAND, + ECALL_THREAD_SETUP, + ECALL_THREAD_CREATE, ECALL_NR, }; diff --git a/Pal/src/host/Linux-SGX/enclave_ecalls.c b/Pal/src/host/Linux-SGX/enclave_ecalls.c index d0c1e0401a..c382127c1f 100644 --- a/Pal/src/host/Linux-SGX/enclave_ecalls.c +++ b/Pal/src/host/Linux-SGX/enclave_ecalls.c @@ -17,6 +17,24 @@ void pal_start_thread (void); extern void * enclave_base, * enclave_top; +struct thread_map { + unsigned int tid; + unsigned int thread_index; + unsigned int status; + sgx_arch_tcs_t * tcs; + unsigned long tcs_addr; + unsigned long ssa_addr; + unsigned long tls_addr; + unsigned long enclave_entry; +}; + +/* pal_expand_stack grows the stack dynamically under EDMM mode, + * the growing strategy is (1) commit EPC pages to the space between + * fault address and the current stack top; (2) commit one more EPC + * page below the fault address for future stack grow + + * fault_addr: the address where causing #PF by push instructions + */ void pal_expand_stack(unsigned long fault_addr) { unsigned long stack_commit_top = GET_ENCLAVE_TLS(stack_commit_top); @@ -25,7 +43,7 @@ void pal_expand_stack(unsigned long fault_addr) unsigned long stack_init_addr = GET_ENCLAVE_TLS(initial_stack_offset); unsigned long end_addr = fault_addr - PRESET_PAGESIZE; - SGX_DBG(DBG_E, "fault_addr, stack_commit_top, stack_init_addr: %p, %p, %p\n", + SGX_DBG(DBG_I, "fault_addr, stack_commit_top, stack_init_addr: %p, %p, %p\n", fault_addr, stack_commit_top, stack_init_addr); if (fault_addr < (stack_init_addr - ENCLAVE_STACK_SIZE * PRESET_PAGESIZE)) { SGX_DBG(DBG_E, "stack overrun, stop!\n"); @@ -35,6 +53,7 @@ void pal_expand_stack(unsigned long fault_addr) sgx_accept_pages(accept_flags, fault_addr, stack_commit_top, 0); stack_commit_top = fault_addr; + /* Overgrow one more page */ if (end_addr >= stack_init_addr - ENCLAVE_STACK_SIZE * PRESET_PAGESIZE) { sgx_accept_pages(accept_flags, end_addr, fault_addr, 0); @@ -43,6 +62,66 @@ void pal_expand_stack(unsigned long fault_addr) } +/* This function setup the pages necessary for runing a thread including: + * (1) SSAs (2) TLS (3)TCS (4) Stack + * ecall_args: pointer to the thread-dependent information for setup the new thread + */ +void pal_thread_setup(void * ecall_args){ + struct thread_map * thread_info = (struct thread_map *)ecall_args; + unsigned long regular_flags = SGX_SECINFO_FLAGS_R | SGX_SECINFO_FLAGS_W | + SGX_SECINFO_FLAGS_REG | SGX_SECINFO_FLAGS_PENDING; + SGX_DBG(DBG_I, "the created thread using tcs at %p, tls at %p, ssa at %p\n", + thread_info->tcs_addr, thread_info->tls_addr, thread_info->ssa_addr); + sgx_accept_pages(regular_flags, thread_info->tcs_addr, thread_info->tcs_addr + PRESET_PAGESIZE, 0); + sgx_accept_pages(regular_flags, thread_info->tls_addr, thread_info->tls_addr + PRESET_PAGESIZE, 0); + sgx_accept_pages(regular_flags, thread_info->ssa_addr, thread_info->ssa_addr + 2 * PRESET_PAGESIZE, 0); + + // Setup TLS + struct enclave_tls* tls = (struct enclave_tls *) thread_info->tls_addr; + tls->enclave_size = GET_ENCLAVE_TLS(enclave_size); + tls->tcs_offset = thread_info->tcs_addr; + + unsigned long stack_gap = thread_info->thread_index * (ENCLAVE_STACK_SIZE + PRESET_PAGESIZE); // There is a gap between stacks + tls->initial_stack_offset = GET_ENCLAVE_TLS(initial_stack_offset) - stack_gap; + + tls->ssa = (void *)thread_info->ssa_addr; + tls->gpr = tls->ssa + PRESET_PAGESIZE - sizeof(sgx_arch_gpr_t); + tls->aux_stack_offset = GET_ENCLAVE_TLS(aux_stack_offset); + tls->stack_commit_top = tls->initial_stack_offset; + tls->ocall_pending = 0; + + // Setup TCS + thread_info->tcs = (sgx_arch_tcs_t *) thread_info->tcs_addr; + memset((void*)thread_info->tcs_addr, 0, PRESET_PAGESIZE); + thread_info->tcs->ossa = thread_info->ssa_addr; + thread_info->tcs->nssa = 2; + thread_info->tcs->oentry = thread_info->enclave_entry; + thread_info->tcs->ofsbasgx = 0; + thread_info->tcs->ogsbasgx = thread_info->tls_addr; + thread_info->tcs->fslimit = 0xfff; + thread_info->tcs->gslimit = 0xfff; + + // PRE-ALLOCATE two pages for STACK + unsigned long accept_flags = SGX_SECINFO_FLAGS_R | SGX_SECINFO_FLAGS_W | + SGX_SECINFO_FLAGS_REG | SGX_SECINFO_FLAGS_PENDING; + + sgx_accept_pages(accept_flags, tls->initial_stack_offset - 2 * PRESET_PAGESIZE, tls->initial_stack_offset, 0); +} + +/* pal_thread_create finalizes the creataion of thread by changing + * the type of tcs page from regular to TCS + * ecall_args: the tcs page address to be TCS type + */ +void pal_thread_create(void * ecall_args){ + struct thread_map * thread_info = (struct thread_map *)ecall_args; + unsigned long tcs_flags = SGX_SECINFO_FLAGS_TCS | SGX_SECINFO_FLAGS_MODIFIED; + + int rs = sgx_accept_pages(tcs_flags, thread_info->tcs_addr, thread_info->tcs_addr + PRESET_PAGESIZE, 0); + if (rs != 0) SGX_DBG(DBG_E, "EACCEPT TCS Change failed: %d\n", rs); +} + + + int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, void * untrusted_stack, void * enclave_base_addr) { @@ -64,8 +143,7 @@ int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, SET_ENCLAVE_TLS(ustack_top, untrusted_stack); SET_ENCLAVE_TLS(ustack, untrusted_stack); SET_ENCLAVE_TLS(ocall_pending, 0); - void * utr_stack = GET_ENCLAVE_TLS(ustack); - SGX_DBG(DBG_E, "utrusted stack: %p\n", utr_stack); + switch(ecall_index) { case ECALL_ENCLAVE_START: { ms_ecall_enclave_start_t * ms = @@ -84,11 +162,15 @@ int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, ocall_exit(); break; case ECALL_STACK_EXPAND: - pal_expand_stack(ecall_args); + pal_expand_stack((unsigned long)ecall_args); + break; + case ECALL_THREAD_SETUP: + pal_thread_setup(ecall_args); + break; + case ECALL_THREAD_CREATE: + pal_thread_create(ecall_args); break; } - utr_stack = GET_ENCLAVE_TLS(ustack); - SGX_DBG(DBG_E, "untrusted stack after: %p\n", utr_stack); - SGX_DBG(DBG_E, "exit_target: %p\n", GET_ENCLAVE_TLS(exit_target)); + return 0; } diff --git a/Pal/src/host/Linux-SGX/enclave_entry.S b/Pal/src/host/Linux-SGX/enclave_entry.S index 24a251310e..8fb72859e9 100644 --- a/Pal/src/host/Linux-SGX/enclave_entry.S +++ b/Pal/src/host/Linux-SGX/enclave_entry.S @@ -11,10 +11,6 @@ enclave_entry: # On EENTER/ERESUME, RAX is the current SSA, RBX is the address of TCS, # RCX is the address of AEP. Other registers are not trusted. - # current SSA is in RAX (Trusted) - # cmp $0, %rax - # jne .Lhandle_resume - # current SSA is in RAX (Trusted) cmp $0, %rax je .Lnormal_enter @@ -23,7 +19,7 @@ enclave_entry: cmp $2, %rdi je .Lnormal_enter - # OCALL made in Exception context + # Not OCALL made in Exception context cmp $1, %gs:SGX_OCALL_PENDING jne .Lhandle_resume @@ -48,11 +44,6 @@ enclave_entry: # from a OCALL in the untrusted PAL. Attackers can manipulate RDI # to deceive the trusted PAL. - # A safe design: check if %gs:SGX_EXIT_TARGET is ever assigned -# mov %gs:SGX_EXIT_TARGET, %rcx -# cmp $0, %rcx -# jne .Lreturn_from_ocall - cmp $0, %gs:SGX_OCALL_PENDING jne .Lreturn_from_ocall @@ -71,10 +62,17 @@ enclave_entry: # push untructed stack address to RCX mov %rsp, %rcx - # handling stack grow, using exception stack + # handle stack grow, using auxiliary stack cmp $2, %rdi - je .Lhandle_exception_stack + je .Lhandle_aux_stack + # setup thread context, using auxiliary stack + cmp $3, %rdi + je .Lhandle_aux_stack + + cmp $4, %rdi + je .Lhandle_aux_stack + # switch to enclve stack: enclave base + %gs:SGX_INITIAL_STACK_OFFSET add %gs:SGX_INITIAL_STACK_OFFSET, %rbx mov %rbx, %rsp @@ -95,13 +93,14 @@ enclave_entry: # to handle_ecall() in enclave_ecalls.c call handle_ecall - # never return to this point (should die) + # ecall_enclave_start/ecall_thread_start never return to this point + # other ecalls return here xor %rdi, %rdi xor %rsi, %rsi jmp .Leexit -.Lhandle_exception_stack: - add %gs:SGX_EXCEPTION_STACK_OFFSET, %rbx +.Lhandle_aux_stack: + add %gs:SGX_AUX_STACK_OFFSET, %rbx mov %rbx, %rsp # In exception handler, ecall returns to the next @@ -266,7 +265,8 @@ sgx_ocall: push %rbp mov %rsp, %gs:SGX_STACK - + + # ocall sets ocall_pending before exit movq $1, %gs:SGX_OCALL_PENDING jmp .Leexit @@ -285,13 +285,15 @@ sgx_ocall: xor %rbp, %rbp mov %gs:SGX_USTACK, %rsp -# and $STACK_ALIGN, %rsp + # If it's an ecall return using + # ecall_ret_target as exit target cmp $1, %gs:SGX_OCALL_PENDING jne .Lecall_return_setup mov %gs:SGX_EXIT_TARGET, %rbx mov %gs:SGX_AEP, %rcx + .Lexecute_exit: mov $EEXIT, %rax ENCLU @@ -308,7 +310,7 @@ sgx_ocall: mov %rdi, %rax - # Clear ocall_pending + # clear ocall_pending movq $0, %gs:SGX_OCALL_PENDING # restore FSBASE if necessary diff --git a/Pal/src/host/Linux-SGX/enclave_framework.c b/Pal/src/host/Linux-SGX/enclave_framework.c index b8d27c4e7b..7be2e83ce3 100644 --- a/Pal/src/host/Linux-SGX/enclave_framework.c +++ b/Pal/src/host/Linux-SGX/enclave_framework.c @@ -100,13 +100,16 @@ int sgx_verify_report (sgx_arch_report_t * report) #define SE_DECLSPEC_ALIGN(x) __attribute__((aligned(x))) +/* sgx_accept_pages do EACCEPT on the pages from address lo to address hi */ int sgx_accept_pages(uint64_t sfl, size_t lo, size_t hi, bool executable) { size_t addr = hi; SE_DECLSPEC_ALIGN(sizeof(sgx_arch_secinfo_t)) sgx_arch_secinfo_t si; si.flags = sfl; + for (uint16_t i = 0; i < (sizeof(si.reserved)/sizeof(si.reserved[0])); i++) si.reserved[i] = 0; + SGX_DBG(DBG_M, "sgx_accept_pages: %p - %p, executable: %d \n", lo, hi, executable); SE_DECLSPEC_ALIGN(sizeof(sgx_arch_secinfo_t)) sgx_arch_secinfo_t smi = si; smi.flags |= SGX_SECINFO_FLAGS_X; @@ -118,13 +121,12 @@ int sgx_accept_pages(uint64_t sfl, size_t lo, size_t hi, bool executable) /* FIXME: Need a better handle here, adding the flow for checking multiple EACCEPT on the same page */ if (rc != 0) { -// SGX_DBG(DBG_E, "eaccept fails: %d\n", rc); -// return rc; - continue; - } - if (executable){ - rc = sgx_modpe(&smi, (const void *)addr); +// SGX_DBG(DBG_E, "eaccept fails: %d\n", rc); +// return rc; + continue; } + if (executable) + rc = sgx_modpe(&smi, (const void *)addr); } return 0; } diff --git a/Pal/src/host/Linux-SGX/pal_linux.h b/Pal/src/host/Linux-SGX/pal_linux.h index 1c95eed06c..efdb409b0e 100644 --- a/Pal/src/host/Linux-SGX/pal_linux.h +++ b/Pal/src/host/Linux-SGX/pal_linux.h @@ -61,6 +61,8 @@ extern struct pal_linux_state { #define PRESET_PAGESIZE (1 << 12) +#define MAX_THREAD_NUM 16 + #define DEFAULT_BACKLOG 2048 static inline int HOST_FLAGS (int alloc_type, int prot) diff --git a/Pal/src/host/Linux-SGX/sgx_enclave.c b/Pal/src/host/Linux-SGX/sgx_enclave.c index c8d2febea1..202c6c106a 100644 --- a/Pal/src/host/Linux-SGX/sgx_enclave.c +++ b/Pal/src/host/Linux-SGX/sgx_enclave.c @@ -726,6 +726,18 @@ int ecall_stack_expand(void * addr) return sgx_ecall(ECALL_STACK_EXPAND, addr); } +int ecall_thread_setup (void * thread_info) +{ + EDEBUG(ECALL_THREAD_SETUP, thread_info); + return sgx_ecall(ECALL_THREAD_SETUP, thread_info); +} + +int ecall_thread_create(void * thread_info) +{ + return sgx_ecall(ECALL_THREAD_CREATE, thread_info); +} + + void __abort(void) { INLINE_SYSCALL(exit_group, 1, -1); } diff --git a/Pal/src/host/Linux-SGX/sgx_enclave.h b/Pal/src/host/Linux-SGX/sgx_enclave.h index 3a05149c80..cd90cb5ca8 100644 --- a/Pal/src/host/Linux-SGX/sgx_enclave.h +++ b/Pal/src/host/Linux-SGX/sgx_enclave.h @@ -9,3 +9,7 @@ int ecall_enclave_start (const char ** arguments, const char ** environments); int ecall_thread_start (void); int ecall_stack_expand (void * fault_addr); + +int ecall_thread_setup (void * thread_info); + +int ecall_thread_create(void * thread_info); diff --git a/Pal/src/host/Linux-SGX/sgx_entry.S b/Pal/src/host/Linux-SGX/sgx_entry.S index 84e4427298..c75c7f90cb 100644 --- a/Pal/src/host/Linux-SGX/sgx_entry.S +++ b/Pal/src/host/Linux-SGX/sgx_entry.S @@ -7,6 +7,14 @@ .type sgx_ecall, @function sgx_ecall: + push %rbp + mov %rsp,%rbp + + # Save registers + push %rbx + push %rcx + push %rdx + # put entry address in RDX lea sgx_entry(%rip), %rdx @@ -23,6 +31,12 @@ sgx_ecall: mov $EENTER, %rax ENCLU + # Restore Registers + pop %rdx + pop %rcx + pop %rbx + + pop %rbp retq .global async_exit_pointer diff --git a/Pal/src/host/Linux-SGX/sgx_exception.c b/Pal/src/host/Linux-SGX/sgx_exception.c index d3ae5d772e..4f54b4d472 100644 --- a/Pal/src/host/Linux-SGX/sgx_exception.c +++ b/Pal/src/host/Linux-SGX/sgx_exception.c @@ -265,16 +265,14 @@ static void _DkResumeSighandler (int signum, siginfo_t * info, break; } #if SGX_HAS_FSGSBASE != 0 - unsigned long fault_addr = info->si_addr; + unsigned long fault_addr = (unsigned long)(info->si_addr); unsigned long stack_start_addr = current_enclave->stackinfo.start_addr; unsigned long stack_end_addr = current_enclave->stackinfo.end_addr; - // fault happened at stack area - if (current_enclave->pal_sec.edmm_mode - && signum == SIGBUS && rax == ERESUME - && (fault_addr <= stack_start_addr - && fault_addr >= stack_end_addr)){ - ecall_stack_expand(fault_addr); - printf("return from stack expand!\n"); + + /* There is a need to grow stack if it's in stack area with SIGBUS under EDMM */ + if (current_enclave->pal_sec.edmm_mode && (signum == SIGBUS && rax == ERESUME) + && (fault_addr <= stack_start_addr && fault_addr >= stack_end_addr)){ + ecall_stack_expand((void *)fault_addr); } else sgx_raise(event); diff --git a/Pal/src/host/Linux-SGX/sgx_framework.c b/Pal/src/host/Linux-SGX/sgx_framework.c index 18a8faf8a0..2eea3613ee 100644 --- a/Pal/src/host/Linux-SGX/sgx_framework.c +++ b/Pal/src/host/Linux-SGX/sgx_framework.c @@ -297,6 +297,28 @@ int add_pages_to_enclave(sgx_arch_secs_t * secs, return 0; } +/* mktcs sends IOCTL to ask sgx driver to change + * the type of a regular page to TCS type */ +void mktcs(unsigned long tcs_addr) +{ + struct sgx_range params; + memset(¶ms, 0 ,sizeof(struct sgx_range)); + params.start_addr = tcs_addr; + params.nr_pages = 1; + int ret = 0; + + +#if SDK_DRIVER_VERSION >= KERNEL_VERSION(2, 0, 0) + ret = INLINE_SYSCALL(ioctl, 3, isgx_device, SGX_IOC_ENCLAVE_MKTCS, ¶ms); + if (IS_ERR(ret)) { + SGX_DBG(DBG_I, "Enclave MKTCS returned %d\n", ret); + return ; + } +#else + SGX_DBG(DBG_E, "EDMM is not supported by SDK before 2.0\n"); +#endif +} + int init_enclave(sgx_arch_secs_t * secs, sgx_arch_sigstruct_t * sigstruct, sgx_arch_token_t * token) diff --git a/Pal/src/host/Linux-SGX/sgx_internal.h b/Pal/src/host/Linux-SGX/sgx_internal.h index df99be9171..0deb6d1bb1 100644 --- a/Pal/src/host/Linux-SGX/sgx_internal.h +++ b/Pal/src/host/Linux-SGX/sgx_internal.h @@ -107,6 +107,7 @@ int add_pages_to_enclave(sgx_arch_secs_t * secs, enum sgx_page_type type, int prot, bool skip_eextend, const char * comment); +void mktcs(unsigned long tcs_addr); int init_enclave(sgx_arch_secs_t * secs, sgx_arch_sigstruct_t * sigstruct, @@ -126,7 +127,9 @@ void double_async_exit (void); int interrupt_thread (void * tcs); int clone_thread (void); -void create_tcs_mapper (void * tcs_base, unsigned int thread_num); +void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned long tls_base, unsigned long enclave_entry, + unsigned int thread_num, unsigned int max_thread_num); + void map_tcs (unsigned int tid); void unmap_tcs (void); diff --git a/Pal/src/host/Linux-SGX/sgx_main.c b/Pal/src/host/Linux-SGX/sgx_main.c index 93271aef97..86b4705fd4 100644 --- a/Pal/src/host/Linux-SGX/sgx_main.c +++ b/Pal/src/host/Linux-SGX/sgx_main.c @@ -333,7 +333,7 @@ int initialize_enclave (struct pal_enclave * enclave) }; struct mem_area * areas = - __alloca(sizeof(areas[0]) * (10 + enclave->thread_num)); + __alloca(sizeof(areas[0]) * (16 + MAX_THREAD_NUM)); int area_num = 0; #define set_area(_desc, _skip_eextend, _is_binary, _fd, _addr, _size, _prot, _type)\ @@ -349,29 +349,34 @@ int initialize_enclave (struct pal_enclave * enclave) set_area("manifest", false, false, enclave->manifest, 0, ALLOC_ALIGNUP(manifest_size), PROT_READ, SGX_PAGE_REG); + + /* leave the virtual address space for maximum supported thread number under EDMM */ + int allocated_thread_num = enclave->pal_sec.edmm_mode ? MAX_THREAD_NUM : enclave->thread_num; + struct mem_area * ssa_area = set_area("ssa", true, false, -1, 0, - enclave->thread_num * enclave->ssaframesize * SSAFRAMENUM, + allocated_thread_num * enclave->ssaframesize * SSAFRAMENUM, PROT_READ|PROT_WRITE, SGX_PAGE_REG); /* XXX: TCS should be part of measurement */ struct mem_area * tcs_area = - set_area("tcs", true, false, -1, 0, enclave->thread_num * pagesize, + set_area("tcs", true, false, -1, 0, allocated_thread_num * pagesize, 0, SGX_PAGE_TCS); /* XXX: TLS should be part of measurement */ struct mem_area * tls_area = - set_area("tls", true, false, -1, 0, enclave->thread_num * pagesize, + set_area("tls", true, false, -1, 0, allocated_thread_num * pagesize, PROT_READ|PROT_WRITE, SGX_PAGE_REG); /* XXX: the enclave stack should be part of measurement */ struct mem_area * stack_areas = &areas[area_num]; - for (int t = 0 ; t < enclave->thread_num ; t++) + for (int t = 0 ; t < allocated_thread_num ; t++) set_area("stack", true, false, -1, 0, ENCLAVE_STACK_SIZE, PROT_READ|PROT_WRITE, SGX_PAGE_REG); - /* XXX: EDMM one page for exception handler stack */ - struct mem_area * exception_stack_area = NULL; - if (enclave->pal_sec.edmm_mode) - exception_stack_area = - set_area("exception_stack", true, false, -1, 0, pagesize, + + /* XXX: EDMM: allocated one page of auxiliary stack for dynamic stack grow + * and thread creation */ + struct mem_area * aux_stack_area = NULL; + if (enclave->pal_sec.edmm_mode) + aux_stack_area = set_area("aux_stack", true, false, -1, 0, pagesize, PROT_READ|PROT_WRITE, SGX_PAGE_REG); struct mem_area * pal_area = @@ -440,7 +445,8 @@ int initialize_enclave (struct pal_enclave * enclave) PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - for (int t = 0 ; t < enclave->thread_num ; t++) { + int tls_thread_num = enclave->pal_sec.edmm_mode ? enclave->thread_num + 1: enclave->thread_num; + for (int t = 0 ; t < tls_thread_num ; t++) { struct enclave_tls * gs = data + pagesize * t; gs->enclave_size = enclave->size; gs->tcs_offset = tcs_area->addr + pagesize * t; @@ -456,10 +462,8 @@ int initialize_enclave (struct pal_enclave * enclave) if (enclave->pal_sec.edmm_mode){ /* TODO: each thread should have their own exception stack */ - gs->exception_stack_offset = exception_stack_area->addr + pagesize; + gs->aux_stack_offset = aux_stack_area->addr + pagesize; gs->stack_commit_top = gs->initial_stack_offset; - //gs->thread_manage_stack_offset = thread_management_stack_area->addr + pagesize; - //printf("gs->exception_stack_offset: 0x%x\n", gs->exception_stack_offset); } } @@ -471,8 +475,8 @@ int initialize_enclave (struct pal_enclave * enclave) data = (void *) INLINE_SYSCALL(mmap, 6, NULL, areas[i].size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0); - - for (int t = 0 ; t < enclave->thread_num ; t++) { + int tcs_thread_num = enclave->pal_sec.edmm_mode ? enclave->thread_num + 1 : enclave->thread_num; + for (int t = 0 ; t < tcs_thread_num ; t++) { sgx_arch_tcs_t * tcs = data + pagesize * t; memset(tcs, 0, pagesize); tcs->ossa = ssa_area->addr + @@ -497,22 +501,42 @@ int initialize_enclave (struct pal_enclave * enclave) areas[i].fd, 0); add_pages: - if (!enclave->pal_sec.edmm_mode || (!strcmp_static(areas[i].desc, "free") && - !strcmp_static(areas[i].desc, "stack"))) { + if (enclave->pal_sec.edmm_mode) { + + if(!strcmp_static(areas[i].desc, "free") && + !strcmp_static(areas[i].desc, "stack")) { + + /* EDMM: SSA/TLS/TCS only commit EPC pages for static threads + commit one more thread for dynamically create new thread context */ + unsigned long area_size = 0; + if (strcmp_static(areas[i].desc, "ssa")) + area_size = (enclave->thread_num + 1) * enclave->ssaframesize * SSAFRAMENUM; + else if (strcmp_static(areas[i].desc, "tcs") || strcmp_static(areas[i].desc, "tls")) + area_size = (enclave->thread_num + 1) * pagesize; + else + area_size = areas[i].size; + + TRY(add_pages_to_enclave, + &enclave_secs, (void *) areas[i].addr, data, area_size, + areas[i].type, areas[i].prot, areas[i].skip_eextend, + areas[i].desc); + } + } + else TRY(add_pages_to_enclave, - &enclave_secs, (void *) areas[i].addr, data, areas[i].size, - areas[i].type, areas[i].prot, areas[i].skip_eextend, - areas[i].desc); - } - + &enclave_secs, (void *) areas[i].addr, data, areas[i].size, + areas[i].type, areas[i].prot, areas[i].skip_eextend, + areas[i].desc); if (data) INLINE_SYSCALL(munmap, 2, data, areas[i].size); } TRY(init_enclave, &enclave_secs, &enclave_sigstruct, &enclave_token); - create_tcs_mapper((void *) enclave_secs.baseaddr + tcs_area->addr, - enclave->thread_num); + create_tcs_mapper(ssa_area->addr, enclave_secs.baseaddr + tcs_area->addr, + enclave_secs.baseaddr + tls_area->addr, enclave_entry_addr, + enclave->thread_num, enclave->pal_sec.edmm_mode? MAX_THREAD_NUM : enclave->thread_num); + struct pal_sec * pal_sec = &enclave->pal_sec; diff --git a/Pal/src/host/Linux-SGX/sgx_thread.c b/Pal/src/host/Linux-SGX/sgx_thread.c index e17ae99b14..aacbdbcc27 100644 --- a/Pal/src/host/Linux-SGX/sgx_thread.c +++ b/Pal/src/host/Linux-SGX/sgx_thread.c @@ -16,36 +16,99 @@ __thread struct pal_enclave * current_enclave; __thread sgx_arch_tcs_t * current_tcs; +enum { + TCS_ALLOC = 0, + TCS_UNALLOC, +}; + struct thread_map { - unsigned int tid; - sgx_arch_tcs_t * tcs; + unsigned int tid; + unsigned int thread_index; + unsigned int status; + sgx_arch_tcs_t * tcs; + unsigned long tcs_addr; + unsigned long ssa_addr; + unsigned long tls_addr; + unsigned long enclave_entry; }; static sgx_arch_tcs_t * enclave_tcs; static int enclave_thread_num; +static int enclave_max_thread_num; static struct thread_map * enclave_thread_map; -void create_tcs_mapper (void * tcs_base, unsigned int thread_num) +/* create_tcs_mapper initializes the thread information for each threads + * thread_num: the number of threads statically allocated + * max_thread_num: the maximum number of threads could be allocated under EDMM + */ +void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned long tls_base, unsigned long enclave_entry, + unsigned int thread_num, unsigned int max_thread_num) { - enclave_tcs = tcs_base; - enclave_thread_map = malloc(sizeof(struct thread_map) * thread_num); + enclave_tcs = (sgx_arch_tcs_t*)tcs_base; enclave_thread_num = thread_num; + enclave_max_thread_num = max_thread_num; - for (int i = 0 ; i < thread_num ; i++) { + enclave_thread_map = malloc(sizeof(struct thread_map) * enclave_max_thread_num); + + for (int i = 0 ; i < enclave_max_thread_num ; i++) { enclave_thread_map[i].tid = 0; + enclave_thread_map[i].thread_index = i; + enclave_thread_map[i].tcs = NULL; + enclave_thread_map[i].ssa_addr = ssa_base + i * pagesize * 2; + enclave_thread_map[i].tcs_addr = tcs_base + i * pagesize; + enclave_thread_map[i].tls_addr = tls_base + i * pagesize; + enclave_thread_map[i].enclave_entry = enclave_entry; enclave_thread_map[i].tcs = &enclave_tcs[i]; + + enclave_thread_map[i].status = TCS_UNALLOC; } } +void create_thread_context(struct thread_map * thread_info) +{ + //using management thread for setup newly-created thread context + current_tcs = enclave_thread_map[enclave_thread_num].tcs; + + ecall_thread_setup((void*)thread_info); + + mktcs(thread_info->tcs_addr); + + ecall_thread_create((void*)thread_info); +} + void map_tcs (unsigned int tid) { - for (int i = 0 ; i < enclave_thread_num ; i++) + for (int i = 0 ; i < enclave_thread_num ; i++){ if (!enclave_thread_map[i].tid) { enclave_thread_map[i].tid = tid; current_tcs = enclave_thread_map[i].tcs; ((struct enclave_dbginfo *) DBGINFO_ADDR)->thread_tids[i] = tid; - break; + return ; + } + } + + /* EDMM Create thread dynamically after static threads run out + * There is one thread at enclave_thead_map[enclave_thread_num] + * which is dedicated as management thread for creating new threads + * start to create threads with enclave_thread_map[enclave_thread_num + 1] + */ + for (int i = enclave_thread_num + 1; i < enclave_max_thread_num; i++){ + if (!enclave_thread_map[i].tid){ + printf("enclave_thread_map[%d].tcs_addr: %p\n", i, enclave_thread_map[i].tcs_addr); + + /* Allocate the thread context (SSA/TLS/TCS) for new + * thread if not allocated previously */ + if (enclave_thread_map[i].status == TCS_UNALLOC) { + // TODO: Add Mutext here + create_thread_context(enclave_thread_map + i); + enclave_thread_map[i].status = TCS_ALLOC; + } + enclave_thread_map[i].tid = tid; + current_tcs = enclave_thread_map[i].tcs; + ((struct enclave_dbginfo *) DBGINFO_ADDR)->thread_tids[i] = tid; + return ; } + } } void unmap_tcs (void) diff --git a/Pal/src/host/Linux-SGX/sgx_tls.h b/Pal/src/host/Linux-SGX/sgx_tls.h index 149b341758..3c075c8966 100644 --- a/Pal/src/host/Linux-SGX/sgx_tls.h +++ b/Pal/src/host/Linux-SGX/sgx_tls.h @@ -19,10 +19,9 @@ struct enclave_tls { void * ustack_top; void * ustack; void * thread; - uint64_t exception_stack_offset; + uint64_t aux_stack_offset; uint64_t stack_commit_top; uint64_t ecall_ret_target; - uint64_t thread_manage_stack_offset; uint64_t ocall_pending; }; @@ -61,10 +60,9 @@ extern uint64_t dummy_debug_variable; #define SGX_USTACK_TOP 0x48 #define SGX_USTACK 0x50 #define SGX_THREAD 0x58 -#define SGX_EXCEPTION_STACK_OFFSET 0x60 -#define SGX_ECALL_RET_TARGET 0x70 -#define SGX_THREAD_MANAGE_STACK_OFFSET 0x78 -#define SGX_OCALL_PENDING 0x80 +#define SGX_AUX_STACK_OFFSET 0x60 +#define SGX_ECALL_RET_TARGET 0x70 +#define SGX_OCALL_PENDING 0x78 #endif #endif /* __SGX_TLS_H__ */ diff --git a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign index 5b5c0181a9..4f1933c501 100755 --- a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign +++ b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign @@ -26,6 +26,7 @@ DEFAULT_ENCLAVE_SIZE = '256M' DEFAULT_THREAD_NUM = 4 ENCLAVE_HEAP_MIN = 0x10000 +MAX_THREAD_NUM = 16 """ Utilities """ def roundup(addr): @@ -303,11 +304,12 @@ def get_loadcmds(filename): return loadcmds class MemoryArea: - def __init__(self, desc, file=None, addr=None, size=None, flags=None): + def __init__(self, desc, file=None, addr=None, size=None, eadd_size=None, flags=None): self.desc = desc self.file = file self.addr = addr self.size = size + self.eadd_size = eadd_size self.flags = flags self.is_binary = False @@ -328,6 +330,7 @@ class MemoryArea: self.addr = mapaddr else: self.size = os.stat(file).st_size + self.eadd_size = self.size if self.addr is not None: self.addr = rounddown(self.addr) @@ -336,21 +339,31 @@ class MemoryArea: def get_memory_areas(manifest, attr, args): edmm_mode = attr['edmm_mode'] + allocated_thread_num = MAX_THREAD_NUM if edmm_mode else attr['thread_num'] + + eadd_thread_num = attr['thread_num'] + 1 if edmm_mode else attr['thread_num'] + areas = [] - areas.append(MemoryArea('ssa', size=attr['thread_num'] * SSAFRAMESIZE * SSAFRAMENUM, + + areas.append(MemoryArea('ssa', size=allocated_thread_num * SSAFRAMESIZE * SSAFRAMENUM, + eadd_size= eadd_thread_num * SSAFRAMESIZE * SSAFRAMENUM, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) - areas.append(MemoryArea('tcs', size=attr['thread_num'] * TCSSIZE, + + areas.append(MemoryArea('tcs', size=allocated_thread_num * TCSSIZE, + eadd_size = eadd_thread_num * TCSSIZE, flags=PAGEINFO_TCS)) - areas.append(MemoryArea('tls', size=attr['thread_num'] * PAGESIZE, + + areas.append(MemoryArea('tls', size=allocated_thread_num * PAGESIZE, + eadd_size = eadd_thread_num * TCSSIZE, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) - for t in range(attr['thread_num']): + for t in range(0, allocated_thread_num): areas.append(MemoryArea('stack', size=ENCLAVE_STACK_SIZE, - flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) + eadd_size = 0, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) if edmm_mode: - areas.append(MemoryArea('exception_stack', size=PAGESIZE*1, - flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) + areas.append(MemoryArea('aux_stack', size=PAGESIZE*1, + eadd_size = PAGESIZE * 1, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) areas.append(MemoryArea('pal', file=args['libpal'], flags=PAGEINFO_REG)) @@ -379,13 +392,13 @@ def populate_memory_areas(manifest, attr, areas): if area.addr + area.size < populating: addr = area.addr + area.size free_areas.append(MemoryArea('free', addr=addr, size=populating - addr, - flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_X|PAGEINFO_REG)) + eadd_size = populating - addr, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_X|PAGEINFO_REG)) populating = area.addr if populating > ENCLAVE_HEAP_MIN: free_areas.append(MemoryArea('free', addr=ENCLAVE_HEAP_MIN, size=populating - ENCLAVE_HEAP_MIN, - flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_X|PAGEINFO_REG)) + eadd_size=populating - ENCLAVE_HEAP_MIN, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_X|PAGEINFO_REG)) return areas + free_areas @@ -521,7 +534,7 @@ def generate_measurement(attr, areas): for a in range(area.addr, area.addr + area.size, PAGESIZE): do_eadd(mrenclave, a, area.flags) else: - for a in range(area.addr, area.addr + area.size, PAGESIZE): + for a in range(area.addr, area.addr + area.eadd_size, PAGESIZE): if area.desc != "free" and area.desc != "stack": do_eadd(mrenclave, a, area.flags) print_area(area.addr, area.size, area.flags, area.desc, False) From 5dc8bb8d933637e4cb274b692de283271377bdb1 Mon Sep 17 00:00:00 2001 From: Li Lei Date: Sat, 4 Aug 2018 20:40:03 -0700 Subject: [PATCH 4/8] Partition auxiliary stack area into small stacks for different threads --- Pal/src/host/Linux-SGX/enclave_ecalls.c | 8 +++++--- Pal/src/host/Linux-SGX/pal_linux.h | 5 +++++ Pal/src/host/Linux-SGX/sgx_internal.h | 2 +- Pal/src/host/Linux-SGX/sgx_main.c | 17 ++++++++++------- Pal/src/host/Linux-SGX/sgx_thread.c | 4 +++- Pal/src/host/Linux-SGX/signer/pal-sgx-sign | 5 +++-- 6 files changed, 27 insertions(+), 14 deletions(-) diff --git a/Pal/src/host/Linux-SGX/enclave_ecalls.c b/Pal/src/host/Linux-SGX/enclave_ecalls.c index c382127c1f..4aa0a907b6 100644 --- a/Pal/src/host/Linux-SGX/enclave_ecalls.c +++ b/Pal/src/host/Linux-SGX/enclave_ecalls.c @@ -25,6 +25,7 @@ struct thread_map { unsigned long tcs_addr; unsigned long ssa_addr; unsigned long tls_addr; + unsigned long aux_stack_addr; /* only applicable to EDMM */ unsigned long enclave_entry; }; @@ -43,7 +44,7 @@ void pal_expand_stack(unsigned long fault_addr) unsigned long stack_init_addr = GET_ENCLAVE_TLS(initial_stack_offset); unsigned long end_addr = fault_addr - PRESET_PAGESIZE; - SGX_DBG(DBG_I, "fault_addr, stack_commit_top, stack_init_addr: %p, %p, %p\n", + SGX_DBG(DBG_M, "fault_addr, stack_commit_top, stack_init_addr: %p, %p, %p\n", fault_addr, stack_commit_top, stack_init_addr); if (fault_addr < (stack_init_addr - ENCLAVE_STACK_SIZE * PRESET_PAGESIZE)) { SGX_DBG(DBG_E, "stack overrun, stop!\n"); @@ -70,7 +71,7 @@ void pal_thread_setup(void * ecall_args){ struct thread_map * thread_info = (struct thread_map *)ecall_args; unsigned long regular_flags = SGX_SECINFO_FLAGS_R | SGX_SECINFO_FLAGS_W | SGX_SECINFO_FLAGS_REG | SGX_SECINFO_FLAGS_PENDING; - SGX_DBG(DBG_I, "the created thread using tcs at %p, tls at %p, ssa at %p\n", + SGX_DBG(DBG_M, "the created thread using tcs at %p, tls at %p, ssa at %p\n", thread_info->tcs_addr, thread_info->tls_addr, thread_info->ssa_addr); sgx_accept_pages(regular_flags, thread_info->tcs_addr, thread_info->tcs_addr + PRESET_PAGESIZE, 0); sgx_accept_pages(regular_flags, thread_info->tls_addr, thread_info->tls_addr + PRESET_PAGESIZE, 0); @@ -86,7 +87,8 @@ void pal_thread_setup(void * ecall_args){ tls->ssa = (void *)thread_info->ssa_addr; tls->gpr = tls->ssa + PRESET_PAGESIZE - sizeof(sgx_arch_gpr_t); - tls->aux_stack_offset = GET_ENCLAVE_TLS(aux_stack_offset); + tls->aux_stack_offset = thread_info->aux_stack_addr; +//GET_ENCLAVE_TLS(aux_stack_offset); tls->stack_commit_top = tls->initial_stack_offset; tls->ocall_pending = 0; diff --git a/Pal/src/host/Linux-SGX/pal_linux.h b/Pal/src/host/Linux-SGX/pal_linux.h index efdb409b0e..de2b785d80 100644 --- a/Pal/src/host/Linux-SGX/pal_linux.h +++ b/Pal/src/host/Linux-SGX/pal_linux.h @@ -63,6 +63,11 @@ extern struct pal_linux_state { #define MAX_THREAD_NUM 16 +#define AUX_STACK_SIZE PRESET_PAGESIZE * 2 + +/* 512B per thread for using auxiliary stack */ +#define AUX_STACK_SIZE_PER_THREAD AUX_STACK_SIZE / MAX_THREAD_NUM + #define DEFAULT_BACKLOG 2048 static inline int HOST_FLAGS (int alloc_type, int prot) diff --git a/Pal/src/host/Linux-SGX/sgx_internal.h b/Pal/src/host/Linux-SGX/sgx_internal.h index 0deb6d1bb1..f5d9a5d735 100644 --- a/Pal/src/host/Linux-SGX/sgx_internal.h +++ b/Pal/src/host/Linux-SGX/sgx_internal.h @@ -127,7 +127,7 @@ void double_async_exit (void); int interrupt_thread (void * tcs); int clone_thread (void); -void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned long tls_base, unsigned long enclave_entry, +void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned long tls_base, unsigned long aux_stack_base, unsigned long enclave_entry, unsigned int thread_num, unsigned int max_thread_num); void map_tcs (unsigned int tid); diff --git a/Pal/src/host/Linux-SGX/sgx_main.c b/Pal/src/host/Linux-SGX/sgx_main.c index 86b4705fd4..d14139519a 100644 --- a/Pal/src/host/Linux-SGX/sgx_main.c +++ b/Pal/src/host/Linux-SGX/sgx_main.c @@ -372,13 +372,13 @@ int initialize_enclave (struct pal_enclave * enclave) set_area("stack", true, false, -1, 0, ENCLAVE_STACK_SIZE, PROT_READ|PROT_WRITE, SGX_PAGE_REG); - /* XXX: EDMM: allocated one page of auxiliary stack for dynamic stack grow + /* XXX: EDMM: allocated two pages of auxiliary stack for dynamic stack grow * and thread creation */ struct mem_area * aux_stack_area = NULL; if (enclave->pal_sec.edmm_mode) - aux_stack_area = set_area("aux_stack", true, false, -1, 0, pagesize, + aux_stack_area = set_area("aux_stack", true, false, -1, 0, AUX_STACK_SIZE, PROT_READ|PROT_WRITE, SGX_PAGE_REG); - + struct mem_area * pal_area = set_area("pal", false, true, enclave_image, 0, 0, 0, SGX_PAGE_REG); TRY(scan_enclave_binary, @@ -461,8 +461,9 @@ int initialize_enclave (struct pal_enclave * enclave) gs->ocall_pending = 0; if (enclave->pal_sec.edmm_mode){ - /* TODO: each thread should have their own exception stack */ - gs->aux_stack_offset = aux_stack_area->addr + pagesize; + /* Each thread has its own region of aux stack from the whole aux_stack_area */ + unsigned long aux_stack_init_addr = aux_stack_area->addr + AUX_STACK_SIZE; + gs->aux_stack_offset = aux_stack_init_addr - t * AUX_STACK_SIZE_PER_THREAD; gs->stack_commit_top = gs->initial_stack_offset; } @@ -532,9 +533,11 @@ int initialize_enclave (struct pal_enclave * enclave) } TRY(init_enclave, &enclave_secs, &enclave_sigstruct, &enclave_token); - + create_tcs_mapper(ssa_area->addr, enclave_secs.baseaddr + tcs_area->addr, - enclave_secs.baseaddr + tls_area->addr, enclave_entry_addr, + enclave_secs.baseaddr + tls_area->addr, + enclave->pal_sec.edmm_mode ? aux_stack_area->addr + AUX_STACK_SIZE:0, + enclave_entry_addr, enclave->thread_num, enclave->pal_sec.edmm_mode? MAX_THREAD_NUM : enclave->thread_num); diff --git a/Pal/src/host/Linux-SGX/sgx_thread.c b/Pal/src/host/Linux-SGX/sgx_thread.c index aacbdbcc27..d58d1981d2 100644 --- a/Pal/src/host/Linux-SGX/sgx_thread.c +++ b/Pal/src/host/Linux-SGX/sgx_thread.c @@ -29,6 +29,7 @@ struct thread_map { unsigned long tcs_addr; unsigned long ssa_addr; unsigned long tls_addr; + unsigned long aux_stack_addr; /* only applicable to EDMM */ unsigned long enclave_entry; }; @@ -41,7 +42,7 @@ static struct thread_map * enclave_thread_map; * thread_num: the number of threads statically allocated * max_thread_num: the maximum number of threads could be allocated under EDMM */ -void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned long tls_base, unsigned long enclave_entry, +void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned long tls_base, unsigned long aux_stack_base, unsigned long enclave_entry, unsigned int thread_num, unsigned int max_thread_num) { enclave_tcs = (sgx_arch_tcs_t*)tcs_base; @@ -57,6 +58,7 @@ void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned enclave_thread_map[i].ssa_addr = ssa_base + i * pagesize * 2; enclave_thread_map[i].tcs_addr = tcs_base + i * pagesize; enclave_thread_map[i].tls_addr = tls_base + i * pagesize; + enclave_thread_map[i].aux_stack_addr = aux_stack_base ? aux_stack_base - i * AUX_STACK_SIZE_PER_THREAD: 0; enclave_thread_map[i].enclave_entry = enclave_entry; enclave_thread_map[i].tcs = &enclave_tcs[i]; diff --git a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign index 4f1933c501..428048f1a1 100755 --- a/Pal/src/host/Linux-SGX/signer/pal-sgx-sign +++ b/Pal/src/host/Linux-SGX/signer/pal-sgx-sign @@ -27,6 +27,7 @@ DEFAULT_THREAD_NUM = 4 ENCLAVE_HEAP_MIN = 0x10000 MAX_THREAD_NUM = 16 +AUX_STACK_SIZE = PAGESIZE * 2 """ Utilities """ def roundup(addr): @@ -362,8 +363,8 @@ def get_memory_areas(manifest, attr, args): eadd_size = 0, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) if edmm_mode: - areas.append(MemoryArea('aux_stack', size=PAGESIZE*1, - eadd_size = PAGESIZE * 1, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) + areas.append(MemoryArea('aux_stack', size=AUX_STACK_SIZE, + eadd_size = AUX_STACK_SIZE, flags=PAGEINFO_R|PAGEINFO_W|PAGEINFO_REG)) areas.append(MemoryArea('pal', file=args['libpal'], flags=PAGEINFO_REG)) From a94dc5e63dbd2f688d61ee05f4a0bcf46a6009be Mon Sep 17 00:00:00 2001 From: Li Lei Date: Sat, 4 Aug 2018 20:49:54 -0700 Subject: [PATCH 5/8] minor updates on comments --- Pal/src/host/Linux-SGX/sgx_thread.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Pal/src/host/Linux-SGX/sgx_thread.c b/Pal/src/host/Linux-SGX/sgx_thread.c index d58d1981d2..27f77773e0 100644 --- a/Pal/src/host/Linux-SGX/sgx_thread.c +++ b/Pal/src/host/Linux-SGX/sgx_thread.c @@ -68,7 +68,7 @@ void create_tcs_mapper (unsigned long ssa_base, unsigned long tcs_base, unsigned void create_thread_context(struct thread_map * thread_info) { - //using management thread for setup newly-created thread context + /* using management thread for setup newly-created thread context */ current_tcs = enclave_thread_map[enclave_thread_num].tcs; ecall_thread_setup((void*)thread_info); @@ -89,7 +89,7 @@ void map_tcs (unsigned int tid) } } - /* EDMM Create thread dynamically after static threads run out + /* EDMM create thread dynamically after static threads run out * There is one thread at enclave_thead_map[enclave_thread_num] * which is dedicated as management thread for creating new threads * start to create threads with enclave_thread_map[enclave_thread_num + 1] @@ -101,7 +101,7 @@ void map_tcs (unsigned int tid) /* Allocate the thread context (SSA/TLS/TCS) for new * thread if not allocated previously */ if (enclave_thread_map[i].status == TCS_UNALLOC) { - // TODO: Add Mutext here + // TODO: Potential race in map_tcs? need a mutex here? create_thread_context(enclave_thread_map + i); enclave_thread_map[i].status = TCS_ALLOC; } From 2def4a79653cc9819f83280f4f4c27cb70fc0538 Mon Sep 17 00:00:00 2001 From: Li Lei Date: Tue, 7 Aug 2018 22:16:33 -0700 Subject: [PATCH 6/8] Fixing some code format --- Pal/src/host/Linux-SGX/enclave_ecalls.c | 13 +-- Pal/src/host/Linux-SGX/enclave_entry.S | 108 ++++++++++++------------ Pal/src/host/Linux-SGX/sgx_enclave.c | 7 +- Pal/src/host/Linux-SGX/sgx_entry.S | 10 +-- Pal/src/host/Linux-SGX/sgx_exception.c | 2 +- Pal/src/host/Linux-SGX/sgx_framework.c | 4 +- 6 files changed, 73 insertions(+), 71 deletions(-) diff --git a/Pal/src/host/Linux-SGX/enclave_ecalls.c b/Pal/src/host/Linux-SGX/enclave_ecalls.c index 4aa0a907b6..6820b5dcba 100644 --- a/Pal/src/host/Linux-SGX/enclave_ecalls.c +++ b/Pal/src/host/Linux-SGX/enclave_ecalls.c @@ -88,7 +88,6 @@ void pal_thread_setup(void * ecall_args){ tls->ssa = (void *)thread_info->ssa_addr; tls->gpr = tls->ssa + PRESET_PAGESIZE - sizeof(sgx_arch_gpr_t); tls->aux_stack_offset = thread_info->aux_stack_addr; -//GET_ENCLAVE_TLS(aux_stack_offset); tls->stack_commit_top = tls->initial_stack_offset; tls->ocall_pending = 0; @@ -122,8 +121,7 @@ void pal_thread_create(void * ecall_args){ if (rs != 0) SGX_DBG(DBG_E, "EACCEPT TCS Change failed: %d\n", rs); } - - +/* handle_ecall is the main entry of all ecall functions */ int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, void * untrusted_stack, void * enclave_base_addr) { @@ -164,14 +162,17 @@ int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, ocall_exit(); break; case ECALL_STACK_EXPAND: - pal_expand_stack((unsigned long)ecall_args); + pal_expand_stack((unsigned long)ecall_args); break; case ECALL_THREAD_SETUP: - pal_thread_setup(ecall_args); + pal_thread_setup(ecall_args); break; case ECALL_THREAD_CREATE: - pal_thread_create(ecall_args); + pal_thread_create(ecall_args); break; + default: + SGX_DBG(DEBUG_E, "Ecall error, invalid ecall index!\n"); + ocall_exit(); } return 0; diff --git a/Pal/src/host/Linux-SGX/enclave_entry.S b/Pal/src/host/Linux-SGX/enclave_entry.S index 8fb72859e9..1ec36f103b 100644 --- a/Pal/src/host/Linux-SGX/enclave_entry.S +++ b/Pal/src/host/Linux-SGX/enclave_entry.S @@ -11,18 +11,18 @@ enclave_entry: # On EENTER/ERESUME, RAX is the current SSA, RBX is the address of TCS, # RCX is the address of AEP. Other registers are not trusted. - # current SSA is in RAX (Trusted) - cmp $0, %rax - je .Lnormal_enter - + # current SSA is in RAX (Trusted) + cmp $0, %rax + je .Lnormal_enter + # Exception for growing stacks - cmp $2, %rdi - je .Lnormal_enter - - # Not OCALL made in Exception context - cmp $1, %gs:SGX_OCALL_PENDING - jne .Lhandle_resume + cmp $2, %rdi + je .Lnormal_enter + # Not OCALL made in Exception context + cmp $1, %gs:SGX_OCALL_PENDING + jne .Lhandle_resume + # TCS is in RBX (Trusted) .Lnormal_enter: @@ -45,7 +45,7 @@ enclave_entry: # to deceive the trusted PAL. cmp $0, %gs:SGX_OCALL_PENDING - jne .Lreturn_from_ocall + jne .Lreturn_from_ocall # PAL convention: # RDI - index in ecall_table @@ -53,7 +53,6 @@ enclave_entry: # RDX - exit target # RCX (former RSP) - The unstrusted stack # R8 - enclave base - # R9 - address of normal return # calculate enclave base = RBX (trusted) - %gs:SGX_TCS_OFFSET sub %gs:SGX_TCS_OFFSET, %rbx @@ -101,13 +100,14 @@ enclave_entry: .Lhandle_aux_stack: add %gs:SGX_AUX_STACK_OFFSET, %rbx - mov %rbx, %rsp - - # In exception handler, ecall returns to the next - # RCX contains the return address, however occupied - # Using SGX_AEP instead - mov %gs:SGX_AEP, %r9 - mov %r9, %gs:SGX_ECALL_RET_TARGET + mov %rbx, %rsp + + # In exception handler, ecall returns to the next + # RCX contains the return address, however occupied + # Using SGX_AEP instead + + mov %gs:SGX_AEP, %r9 + mov %r9, %gs:SGX_ECALL_RET_TARGET jmp .Ldo_handle_ecall .Lhandle_resume: @@ -267,7 +267,7 @@ sgx_ocall: mov %rsp, %gs:SGX_STACK # ocall sets ocall_pending before exit - movq $1, %gs:SGX_OCALL_PENDING + movq $1, %gs:SGX_OCALL_PENDING jmp .Leexit .Lexception_handler: @@ -287,7 +287,7 @@ sgx_ocall: mov %gs:SGX_USTACK, %rsp # If it's an ecall return using - # ecall_ret_target as exit target + # ecall_ret_target as exit target cmp $1, %gs:SGX_OCALL_PENDING jne .Lecall_return_setup @@ -300,8 +300,8 @@ sgx_ocall: .Lecall_return_setup: mov %gs:SGX_ECALL_RET_TARGET, %rbx - mov %gs:SGX_AEP, %rcx - jmp .Lexecute_exit + mov %gs:SGX_AEP, %rcx + jmp .Lexecute_exit .Lreturn_from_ocall: # PAL convention: @@ -311,7 +311,7 @@ sgx_ocall: mov %rdi, %rax # clear ocall_pending - movq $0, %gs:SGX_OCALL_PENDING + movq $0, %gs:SGX_OCALL_PENDING # restore FSBASE if necessary mov %gs:SGX_FSBASE, %rbx @@ -405,47 +405,47 @@ sgx_getkey: * sgx_accept: * EACCEPT pages for dynamic memory management */ - .global sgx_accept - .type sgx_accept, @function -sgx_accept: - .cfi_startproc + .global sgx_accept + .type sgx_accept, @function - push %rbx - push %rcx - mov %rdi, %rbx - mov %rsi, %rcx - mov $EACCEPT, %rax - ENCLU - pop %rcx - pop %rbx - ret +sgx_accept: + .cfi_startproc + + push %rbx + push %rcx + mov %rdi, %rbx + mov %rsi, %rcx + mov $EACCEPT, %rax + ENCLU + pop %rcx + pop %rbx + ret - .cfi_endproc - .size sgx_accept, .-sgx_accept + .cfi_endproc + .size sgx_accept, .-sgx_accept /* * sgx_modpe: * EMODPE pages for dynamic memory management */ + .global sgx_modpe + .type sgx_modpe, @function - .global sgx_modpe - .type sgx_modpe, @function sgx_modpe: - .cfi_startproc - - push %rbx - push %rcx - mov %rdi, %rbx - mov %rsi, %rcx - mov $EMODPE, %rax - ENCLU - pop %rcx - pop %rbx - ret + .cfi_startproc - .cfi_endproc - .size sgx_modpe, .-sgx_modpe + push %rbx + push %rcx + mov %rdi, %rbx + mov %rsi, %rcx + mov $EMODPE, %rax + ENCLU + pop %rcx + pop %rbx + ret + .cfi_endproc + .size sgx_modpe, .-sgx_modpe /* * rdrand: diff --git a/Pal/src/host/Linux-SGX/sgx_enclave.c b/Pal/src/host/Linux-SGX/sgx_enclave.c index 202c6c106a..c8c2fc4302 100644 --- a/Pal/src/host/Linux-SGX/sgx_enclave.c +++ b/Pal/src/host/Linux-SGX/sgx_enclave.c @@ -728,13 +728,14 @@ int ecall_stack_expand(void * addr) int ecall_thread_setup (void * thread_info) { - EDEBUG(ECALL_THREAD_SETUP, thread_info); - return sgx_ecall(ECALL_THREAD_SETUP, thread_info); + EDEBUG(ECALL_THREAD_SETUP, thread_info); + return sgx_ecall(ECALL_THREAD_SETUP, thread_info); } int ecall_thread_create(void * thread_info) { - return sgx_ecall(ECALL_THREAD_CREATE, thread_info); + EDEBUG(ECALL_THREAD_CREATE, thread_info); + return sgx_ecall(ECALL_THREAD_CREATE, thread_info); } diff --git a/Pal/src/host/Linux-SGX/sgx_entry.S b/Pal/src/host/Linux-SGX/sgx_entry.S index c75c7f90cb..96e89fd434 100644 --- a/Pal/src/host/Linux-SGX/sgx_entry.S +++ b/Pal/src/host/Linux-SGX/sgx_entry.S @@ -8,10 +8,10 @@ sgx_ecall: push %rbp - mov %rsp,%rbp + mov %rsp,%rbp # Save registers - push %rbx + push %rbx push %rcx push %rdx @@ -34,10 +34,10 @@ sgx_ecall: # Restore Registers pop %rdx pop %rcx - pop %rbx + pop %rbx - pop %rbp - retq + pop %rbp + retq .global async_exit_pointer .type async_exit_pointer, @function diff --git a/Pal/src/host/Linux-SGX/sgx_exception.c b/Pal/src/host/Linux-SGX/sgx_exception.c index 4f54b4d472..80f89c21bf 100644 --- a/Pal/src/host/Linux-SGX/sgx_exception.c +++ b/Pal/src/host/Linux-SGX/sgx_exception.c @@ -269,7 +269,7 @@ static void _DkResumeSighandler (int signum, siginfo_t * info, unsigned long stack_start_addr = current_enclave->stackinfo.start_addr; unsigned long stack_end_addr = current_enclave->stackinfo.end_addr; - /* There is a need to grow stack if it's in stack area with SIGBUS under EDMM */ + /* need to grow stack if it's in stack area with SIGBUS under EDMM */ if (current_enclave->pal_sec.edmm_mode && (signum == SIGBUS && rax == ERESUME) && (fault_addr <= stack_start_addr && fault_addr >= stack_end_addr)){ ecall_stack_expand((void *)fault_addr); diff --git a/Pal/src/host/Linux-SGX/sgx_framework.c b/Pal/src/host/Linux-SGX/sgx_framework.c index 2eea3613ee..e2cf537b57 100644 --- a/Pal/src/host/Linux-SGX/sgx_framework.c +++ b/Pal/src/host/Linux-SGX/sgx_framework.c @@ -177,7 +177,7 @@ int create_enclave(sgx_arch_secs_t * secs, #endif if (IS_ERR(ret)) { - SGX_DBG(DBG_E, "enclave ECREATE failed in enclave creation ioctl - %d\n", ERRNO(ret)); + SGX_DBG(DBG_I, "enclave ECREATE failed in enclave creation ioctl - %d\n", ERRNO(ret)); return -ERRNO(ret); } @@ -311,7 +311,7 @@ void mktcs(unsigned long tcs_addr) #if SDK_DRIVER_VERSION >= KERNEL_VERSION(2, 0, 0) ret = INLINE_SYSCALL(ioctl, 3, isgx_device, SGX_IOC_ENCLAVE_MKTCS, ¶ms); if (IS_ERR(ret)) { - SGX_DBG(DBG_I, "Enclave MKTCS returned %d\n", ret); + SGX_DBG(DBG_I, "Enclave MKTCS returned %d\n", ret); return ; } #else From e5094c50649dc10b395cd23be7849578396f07db Mon Sep 17 00:00:00 2001 From: Li Lei Date: Tue, 7 Aug 2018 23:07:36 -0700 Subject: [PATCH 7/8] a type fix --- Pal/src/host/Linux-SGX/enclave_ecalls.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Pal/src/host/Linux-SGX/enclave_ecalls.c b/Pal/src/host/Linux-SGX/enclave_ecalls.c index 6820b5dcba..48b55fe371 100644 --- a/Pal/src/host/Linux-SGX/enclave_ecalls.c +++ b/Pal/src/host/Linux-SGX/enclave_ecalls.c @@ -171,7 +171,7 @@ int handle_ecall (long ecall_index, void * ecall_args, void * exit_target, pal_thread_create(ecall_args); break; default: - SGX_DBG(DEBUG_E, "Ecall error, invalid ecall index!\n"); + SGX_DBG(DBG_E, "Ecall error, invalid ecall index!\n"); ocall_exit(); } From 92ecf2c21e2220593b1ad6a80193d4531443f195 Mon Sep 17 00:00:00 2001 From: Li Lei Date: Tue, 7 Aug 2018 23:24:08 -0700 Subject: [PATCH 8/8] Fixing sgx driver compatibility issue --- Pal/src/host/Linux-SGX/sgx_framework.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Pal/src/host/Linux-SGX/sgx_framework.c b/Pal/src/host/Linux-SGX/sgx_framework.c index e2cf537b57..98d96195b6 100644 --- a/Pal/src/host/Linux-SGX/sgx_framework.c +++ b/Pal/src/host/Linux-SGX/sgx_framework.c @@ -301,14 +301,13 @@ int add_pages_to_enclave(sgx_arch_secs_t * secs, * the type of a regular page to TCS type */ void mktcs(unsigned long tcs_addr) { +#if SDK_DRIVER_VERSION == KERNEL_VERSION(2, 0, 0) struct sgx_range params; memset(¶ms, 0 ,sizeof(struct sgx_range)); params.start_addr = tcs_addr; params.nr_pages = 1; int ret = 0; - -#if SDK_DRIVER_VERSION >= KERNEL_VERSION(2, 0, 0) ret = INLINE_SYSCALL(ioctl, 3, isgx_device, SGX_IOC_ENCLAVE_MKTCS, ¶ms); if (IS_ERR(ret)) { SGX_DBG(DBG_I, "Enclave MKTCS returned %d\n", ret);