diff --git a/.gitmodules b/.gitmodules index 49b10248a..824d77df9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "test/unit/objs"] path = test/unit/objs - url = https://github.com/dynup/kpatch-unit-test-objs.git + url = https://github.com/swine/kpatch-unit-test-objs.git + branch = remotes/github-swine/arm64 diff --git a/README.md b/README.md index 75995fe8c..b9c563000 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Supported Architectures - [x] x86-64 - [x] ppc64le -- [ ] arm64 +- [x] arm64 [upstream prerequisites](doc/arm64-upstream-prerequisites.md) - [x] s390 [upstream prerequisites](doc/s390-upstream-prerequisites.md) Installation diff --git a/doc/arm64-upstream-prerequisites.md b/doc/arm64-upstream-prerequisites.md new file mode 100644 index 000000000..3c49af860 --- /dev/null +++ b/doc/arm64-upstream-prerequisites.md @@ -0,0 +1,11 @@ +### arm64 backporting + +**Prerequisite kernel patches:** +**v5.19:** +- [Madhavan Venkataraman's [RFC PATCH v2 00/20] arm64: livepatch: Use ORC for dynamic frame pointer validation](https://lore.kernel.org/linux-arm-kernel/20220524001637.1707472-1-madvenka@linux.microsoft.com/) +- also tested against madvenka's earlier pre-objtool series up to v15 + +**v5.15 and v5.10:** +- under development, both known to work with backports of madvenka's v15, + but the objtool-using version above is likely to be the approach that + finally merges into upstream kernel diff --git a/kmod/patch/kpatch-syscall.h b/kmod/patch/kpatch-syscall.h index ec8d24d53..84b90157c 100644 --- a/kmod/patch/kpatch-syscall.h +++ b/kmod/patch/kpatch-syscall.h @@ -186,7 +186,34 @@ # endif /* LINUX_VERSION_CODE */ -#endif /* CONFIG_X86_64 */ +#elif defined(CONFIG_ARM64) + +/* arm64/include/asm/syscall_wrapper.h versions */ + +#define SC_ARM64_REGS_TO_ARGS(x, ...) \ + __MAP(x,__SC_ARGS \ + ,,regs->regs[0],,regs->regs[1],,regs->regs[2] \ + ,,regs->regs[3],,regs->regs[4],,regs->regs[5]) + +#define __KPATCH_SYSCALL_DEFINEx(x, name, ...) \ + asmlinkage long __arm64_sys##name(const struct pt_regs *regs); \ + ALLOW_ERROR_INJECTION(__arm64_sys##name, ERRNO); \ + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \ + static inline long __kpatch_do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \ + asmlinkage long __arm64_sys##name(const struct pt_regs *regs) \ + { \ + return __se_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \ + } \ + static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \ + { \ + long ret = __kpatch_do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__)); \ + __MAP(x,__SC_TEST,__VA_ARGS__); \ + __PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__)); \ + return ret; \ + } \ + static inline long __kpatch_do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) + +#endif /* which arch */ #ifndef __KPATCH_SYSCALL_DEFINEx diff --git a/kpatch-build/Makefile b/kpatch-build/Makefile index bebf3cd96..7fb223138 100644 --- a/kpatch-build/Makefile +++ b/kpatch-build/Makefile @@ -22,7 +22,7 @@ PLUGIN_CFLAGS := $(filter-out -Wconversion, $(CFLAGS)) PLUGIN_CFLAGS += -shared -I$(GCC_PLUGINS_DIR)/include \ -Igcc-plugins -fPIC -fno-rtti -O2 -Wall endif -ifeq ($(filter $(ARCH),s390x x86_64 ppc64le),) +ifeq ($(filter $(ARCH),aarch64 s390x x86_64 ppc64le),) $(error Unsupported architecture ${ARCH}, check https://github.com/dynup/kpatch/#supported-architectures) endif diff --git a/kpatch-build/create-diff-object.c b/kpatch-build/create-diff-object.c index 25710e921..185b54a50 100644 --- a/kpatch-build/create-diff-object.c +++ b/kpatch-build/create-diff-object.c @@ -173,6 +173,8 @@ static bool is_gcc6_localentry_bundled_sym(struct kpatch_elf *kelf, struct symbol *sym) { switch(kelf->arch) { + case AARCH64: + return false; case PPC64: return ((PPC64_LOCAL_ENTRY_OFFSET(sym->sym.st_other) != 0) && sym->sym.st_value == 8); @@ -215,6 +217,8 @@ static bool is_gcc6_localentry_bundled_sym(struct kpatch_elf *kelf, */ static struct rela *toc_rela(const struct rela *rela) { + if (!is_arch(PPC64)) + return (struct rela *)rela; if (rela->type != R_PPC64_TOC16_HA && rela->type != R_PPC64_TOC16_LO_DS) return (struct rela *)rela; @@ -228,6 +232,24 @@ static struct rela *toc_rela(const struct rela *rela) (unsigned int)rela->addend); } +/* + * Mapping symbols are used to mark and label the transitions between code and + * data in elf files. They begin with a "$" dollar symbol. Don't correlate them + * as they often all have the same name either "$x" to mark the start of code + * or "$d" to mark the start of data. + */ +static bool kpatch_is_mapping_symbol(struct kpatch_elf *kelf, struct symbol *sym) +{ + if (kelf->arch != AARCH64) + return false; + + if (sym->name && sym->name[0] == '$' && + sym->type == STT_NOTYPE && + sym->bind == STB_LOCAL) + return true; + return false; +} + /* * When compiling with -ffunction-sections and -fdata-sections, almost every * symbol gets its own dedicated section. We call such symbols "bundled" @@ -331,6 +353,28 @@ static bool is_string_literal_section(struct section *sec) return !strncmp(sec->name, ".rodata.", 8) && strstr(sec->name, ".str"); } +/* gcc's ".data.unlikely" or clang's ".(data|bss).module_name.unlikely" */ +static bool is_data_unlikely_section(const char *name) +{ + size_t len = strlen(name); + + return (len >= 5 + 8 && + ((!strncmp(name, ".data.", 6) || + !strncmp(name, ".bss.", 5)) && + strstr(name + len - 9, ".unlikely"))); +} + +/* either ".data.once" or clang's ".(data|bss).module_name.once" */ +static bool is_data_once_section(const char *name) +{ + size_t len = strlen(name); + + return (len >= 5 + 4 && + (!strncmp(name, ".data.", 6) || + !strncmp(name, ".bss.", 5)) && + strstr(name + len - 5, ".once")); +} + /* * This function detects whether the given symbol is a "special" static local * variable (for lack of a better term). @@ -372,7 +416,7 @@ static bool is_special_static(struct symbol *sym) if (sym->type != STT_OBJECT || sym->bind != STB_LOCAL) return false; - if (!strcmp(sym->sec->name, ".data.once")) + if (is_data_once_section(sym->sec->name)) return true; for (var_name = var_names; *var_name; var_name++) { @@ -622,6 +666,13 @@ static void kpatch_compare_correlated_section(struct section *sec) goto out; } + /* As above but for __p_f_e users like aarch64 */ + if (!strcmp(sec->name, ".rela__patchable_function_entries") || + !strcmp(sec->name, "__patchable_function_entries")) { + sec->status = SAME; + goto out; + } + if (sec1->sh.sh_size != sec2->sh.sh_size || sec1->data->d_size != sec2->data->d_size || (sec1->rela && !sec2->rela) || @@ -700,6 +751,12 @@ static bool insn_is_load_immediate(struct kpatch_elf *kelf, void *addr) break; + case AARCH64: + /* Verify mov w2 */ + if ((insn[0] & 0b11111) == 0x2 && insn[3] == 0x52) + return true; + break; + default: ERROR("unsupported arch"); } @@ -740,6 +797,7 @@ static bool kpatch_line_macro_change_only(struct kpatch_elf *kelf, void *data1, *data2, *insn1, *insn2; struct rela *r, *rela; bool found, found_any = false; + bool warn_printk_only = (kelf->arch == AARCH64); if (sec->status != CHANGED || is_rela_section(sec) || @@ -803,8 +861,15 @@ static bool kpatch_line_macro_change_only(struct kpatch_elf *kelf, !strncmp(rela->sym->name, "__func__.", 9)) continue; + if (!strcmp(rela->sym->name, "__warn_printk")) { + found = true; + break; + } + + if (warn_printk_only) + return false; + if (!strncmp(rela->sym->name, "warn_slowpath_", 14) || - !strcmp(rela->sym->name, "__warn_printk") || !strcmp(rela->sym->name, "__might_sleep") || !strcmp(rela->sym->name, "___might_sleep") || !strcmp(rela->sym->name, "__might_fault") || @@ -1069,15 +1134,15 @@ static void kpatch_correlate_sections(struct list_head *seclist_orig, } } -static void kpatch_correlate_symbols(struct list_head *symlist_orig, - struct list_head *symlist_patched) +static void kpatch_correlate_symbols(struct kpatch_elf *kelf_orig, + struct kpatch_elf *kelf_patched) { struct symbol *sym_orig, *sym_patched; - list_for_each_entry(sym_orig, symlist_orig, list) { + list_for_each_entry(sym_orig, &kelf_orig->symbols, list) { if (sym_orig->twin) continue; - list_for_each_entry(sym_patched, symlist_patched, list) { + list_for_each_entry(sym_patched, &kelf_patched->symbols, list) { if (kpatch_mangled_strcmp(sym_orig->name, sym_patched->name) || sym_orig->type != sym_patched->type || sym_patched->twin) continue; @@ -1092,9 +1157,14 @@ static void kpatch_correlate_symbols(struct list_head *symlist_orig, * The .LCx symbols point to string literals in * '.rodata..str1.*' sections. They get included * in kpatch_include_standard_elements(). + * Clang creates similar .Ltmp%d symbols in .rodata.str */ if (sym_orig->type == STT_NOTYPE && - !strncmp(sym_orig->name, ".LC", 3)) + !(strncmp(sym_orig->name, ".LC", 3) && + strncmp(sym_orig->name, ".Ltmp", 5))) + continue; + + if (kpatch_is_mapping_symbol(kelf_orig, sym_orig)) continue; /* group section symbols must have correlated sections */ @@ -1502,7 +1572,7 @@ static void kpatch_correlate_elfs(struct kpatch_elf *kelf_orig, struct kpatch_elf *kelf_patched) { kpatch_correlate_sections(&kelf_orig->sections, &kelf_patched->sections); - kpatch_correlate_symbols(&kelf_orig->symbols, &kelf_patched->symbols); + kpatch_correlate_symbols(kelf_orig, kelf_patched); } static void kpatch_compare_correlated_elements(struct kpatch_elf *kelf) @@ -1575,6 +1645,7 @@ static void kpatch_replace_sections_syms(struct kpatch_elf *kelf) !strcmp(rela->sym->name, ".fixup") || !strcmp(rela->sym->name, ".altinstr_replacement") || !strcmp(rela->sym->name, ".altinstr_aux") || + !strncmp(rela->sym->name, ".data..Lubsan", 13) || !strcmp(rela->sym->name, ".text..refcount") || !strncmp(rela->sym->name, "__ftr_alt_", 10)) continue; @@ -1618,7 +1689,8 @@ static void kpatch_replace_sections_syms(struct kpatch_elf *kelf) if (is_text_section(relasec->base) && !is_text_section(sym->sec) && - rela->type == R_X86_64_32S && + ((is_arch(X86_64) && rela->type == R_X86_64_32S) || + (is_arch(AARCH64) && rela->type == R_AARCH64_ABS64)) && rela->addend == (long)sym->sec->sh.sh_size && end == (long)sym->sec->sh.sh_size) { @@ -1655,6 +1727,9 @@ static void kpatch_replace_sections_syms(struct kpatch_elf *kelf) */ } else if (target_off == start && target_off == end) { + if(kpatch_is_mapping_symbol(kelf, sym)) + continue; + /* * Allow replacement for references to * empty symbols. @@ -1694,7 +1769,7 @@ static void kpatch_check_func_profiling_calls(struct kpatch_elf *kelf) (sym->parent && sym->parent->status == CHANGED)) continue; if (!sym->twin->has_func_profiling) { - log_normal("function %s has no fentry/mcount call, unable to patch\n", + log_normal("function %s doesn't have patchable function entry, unable to patch\n", sym->name); errs++; } @@ -1732,8 +1807,11 @@ static void kpatch_verify_patchability(struct kpatch_elf *kelf) * (.data.unlikely and .data.once is ok b/c it only has __warned vars) */ if (sec->include && sec->status != NEW && - (!strncmp(sec->name, ".data", 5) || !strncmp(sec->name, ".bss", 4)) && - (strcmp(sec->name, ".data.unlikely") && strcmp(sec->name, ".data.once"))) { + (!strncmp(sec->name, ".data", 5) || + !strncmp(sec->name, ".bss", 4)) && + !is_data_once_section(sec->name) && + !is_data_unlikely_section(sec->name) && + strncmp(sec->name, ".data..Lubsan", 13)) { log_normal("data section %s selected for inclusion\n", sec->name); errs++; @@ -1829,6 +1907,7 @@ static void kpatch_include_standard_elements(struct kpatch_elf *kelf) !strcmp(sec->name, ".symtab") || !strcmp(sec->name, ".toc") || !strcmp(sec->name, ".rodata") || + !strncmp(sec->name, ".data..Lubsan", 13) || is_string_literal_section(sec)) { kpatch_include_section(sec); } @@ -2415,28 +2494,28 @@ static bool static_call_sites_group_filter(struct lookup_table *lookup, static struct special_section special_sections[] = { { .name = "__bug_table", - .arch = X86_64 | PPC64 | S390, + .arch = AARCH64 | X86_64 | PPC64 | S390, .group_size = bug_table_group_size, }, { .name = ".fixup", - .arch = X86_64 | PPC64 | S390, + .arch = AARCH64 | X86_64 | PPC64 | S390, .group_size = fixup_group_size, }, { .name = "__ex_table", /* must come after .fixup */ - .arch = X86_64 | PPC64 | S390, + .arch = AARCH64 | X86_64 | PPC64 | S390, .group_size = ex_table_group_size, }, { .name = "__jump_table", - .arch = X86_64 | PPC64 | S390, + .arch = AARCH64 | X86_64 | PPC64 | S390, .group_size = jump_table_group_size, .group_filter = jump_table_group_filter, }, { .name = ".printk_index", - .arch = X86_64 | PPC64 | S390, + .arch = AARCH64 | X86_64 | PPC64 | S390, .group_size = printk_index_group_size, }, { @@ -2451,7 +2530,7 @@ static struct special_section special_sections[] = { }, { .name = ".altinstructions", - .arch = X86_64 | S390, + .arch = AARCH64 | X86_64 | S390, .group_size = altinstructions_group_size, }, { @@ -3159,7 +3238,7 @@ static void kpatch_create_patches_sections(struct kpatch_elf *kelf, if (sym->bind == STB_LOCAL && symbol.global) ERROR("can't find local symbol '%s' in symbol table", sym->name); - log_debug("lookup for %s: obj=%s sympos=%lu size=%lu", + log_debug("lookup for %s: obj=%s sympos=%lu size=%lu\n", sym->name, symbol.objname, symbol.sympos, symbol.size); @@ -3236,12 +3315,23 @@ static int function_ptr_rela(const struct rela *rela) { const struct rela *rela_toc = toc_rela(rela); + switch (def_arch()) { + case PPC64: + if (rela->type != R_PPC64_TOC16_HA && + rela->type != R_PPC64_TOC16_LO_DS) + return false; + break; + case X86_64: + if (rela->type != R_X86_64_32S) + return false; + break; + default: + break; + } + return (rela_toc && rela_toc->sym->type == STT_FUNC && !rela_toc->sym->parent && - rela_toc->addend == (int)rela_toc->sym->sym.st_value && - (rela->type == R_X86_64_32S || - rela->type == R_PPC64_TOC16_HA || - rela->type == R_PPC64_TOC16_LO_DS)); + rela_toc->addend == (int)rela_toc->sym->sym.st_value); } static bool need_klp_reloc(struct kpatch_elf *kelf, struct lookup_table *table, @@ -3256,32 +3346,38 @@ static bool need_klp_reloc(struct kpatch_elf *kelf, struct lookup_table *table, * These references are treated specially by the module loader and * should never be converted to klp relocations. */ - if (rela->type == R_PPC64_REL16_HA || rela->type == R_PPC64_REL16_LO || - rela->type == R_PPC64_ENTRY) - return false; + switch (kelf->arch) { + case PPC64: + if (rela->type == R_PPC64_REL16_HA || rela->type == R_PPC64_REL16_LO || + rela->type == R_PPC64_ENTRY) + return false; - /* v5.13+ kernels use relative jump labels */ - if (rela->type == R_PPC64_REL64 && strcmp(relasec->name, ".rela__jump_table")) - return false; + /* v5.13+ kernels use relative jump labels */ + if (rela->type == R_PPC64_REL64 && strcmp(relasec->name, ".rela__jump_table")) + return false; - /* - * On powerpc, the function prologue generated by GCC 6 has the - * sequence: - * - * .globl my_func - * .type my_func, @function - * .quad .TOC.-my_func - * my_func: - * .reloc ., R_PPC64_ENTRY ; optional - * ld r2,-8(r12) - * add r2,r2,r12 - * .localentry my_func, .-my_func - * - * The R_PPC64_ENTRY is optional and its symbol might have an empty - * name. Leave it as a normal rela. - */ - if (rela->type == R_PPC64_ENTRY) - return false; + /* + * On powerpc, the function prologue generated by GCC 6 has the + * sequence: + * + * .globl my_func + * .type my_func, @function + * .quad .TOC.-my_func + * my_func: + * .reloc ., R_PPC64_ENTRY ; optional + * ld r2,-8(r12) + * add r2,r2,r12 + * .localentry my_func, .-my_func + * + * The R_PPC64_ENTRY is optional and its symbol might have an empty + * name. Leave it as a normal rela. + */ + if (rela->type == R_PPC64_ENTRY) + return false; + break; + default: + break; + } /* * Allow references to core module symbols to remain as normal @@ -3531,7 +3627,7 @@ static void kpatch_create_intermediate_sections(struct kpatch_elf *kelf, ERROR("can't find symbol '%s' in symbol table", rela->sym->name); - log_debug("lookup for %s: obj=%s sympos=%lu", + log_debug("lookup for %s: obj=%s sympos=%lu\n", rela->sym->name, symbol.objname, symbol.sympos); @@ -3676,6 +3772,25 @@ static void kpatch_create_callbacks_objname_rela(struct kpatch_elf *kelf, char * } } +static void kpatch_set_pfe_link(struct kpatch_elf *kelf) +{ + struct section* sec; + struct rela *rela; + + list_for_each_entry(sec, &kelf->sections, list) { + if (strcmp(sec->name, "__patchable_function_entries")) { + continue; + } + + if (!sec->rela) { + continue; + } + list_for_each_entry(rela, &sec->rela->relas, list) { + rela->sym->sec->pfe = sec; + } + } +} + /* * This function basically reimplements the functionality of the Linux * recordmcount script, so that patched functions can be recognized by ftrace. @@ -3686,25 +3801,44 @@ static void kpatch_create_callbacks_objname_rela(struct kpatch_elf *kelf, char * static void kpatch_create_mcount_sections(struct kpatch_elf *kelf) { int nr, index; - struct section *sec, *relasec; + struct section *relasec; struct symbol *sym; struct rela *rela, *mcount_rela; void **funcs; - unsigned long insn_offset = 0; + bool pfe_per_function = false; nr = 0; list_for_each_entry(sym, &kelf->symbols, list) if (sym->type == STT_FUNC && sym->status != SAME && - sym->has_func_profiling) + sym->has_func_profiling) nr++; - /* create text/rela section pair */ - sec = create_section_pair(kelf, "__mcount_loc", sizeof(void*), nr); - relasec = sec->rela; + switch (kelf->arch) { + case AARCH64: + /* For aarch64, we will create separate __patchable_function_entries sections for each symbols. */ + pfe_per_function = true; + relasec = NULL; + break; + case PPC64: + case X86_64: + case S390: + { + struct section *sec; + + /* create text/rela section pair */ + sec = create_section_pair(kelf, "__mcount_loc", sizeof(void*), nr); + relasec = sec->rela; + break; + } + default: + ERROR("unsupported arch\n"); + } /* populate sections */ index = 0; list_for_each_entry(sym, &kelf->symbols, list) { + unsigned long insn_offset = 0; + if (sym->type != STT_FUNC || sym->status == SAME) continue; @@ -3715,6 +3849,42 @@ static void kpatch_create_mcount_sections(struct kpatch_elf *kelf) } switch(kelf->arch) { + case AARCH64: { + struct section *sec; + unsigned char *insn; + int i; + + insn = sym->sec->data->d_buf; + + /* + * If BTI (Branch Target Identification) is enabled then there + * might be an additional 'BTI C' instruction before the two + * patchable function entry 'NOP's. + * i.e. 0xd503245f (little endian) + */ + if (insn[0] == 0x5f) { + if (insn[1] != 0x24 || insn[2] != 0x03 || insn[3] != 0xd5) + ERROR("%s: unexpected instruction in patch section of function\n", sym->name); + insn_offset += 4; + insn += 4; + } + for (i = 0; i < 8; i += 4) { + /* We expect a NOP i.e. 0xd503201f (little endian) */ + if (insn[i] != 0x1f || insn[i + 1] != 0x20 || + insn[i + 2] != 0x03 || insn [i + 3] != 0xd5) + ERROR("%s: unexpected instruction in patch section of function\n", sym->name); + } + + /* Allocate __patchable_function_entries for symbol */ + sec = create_section_pair(kelf, "__patchable_function_entries", sizeof(void *), 1); + sec->sh.sh_flags |= SHF_WRITE | SHF_LINK_ORDER; + /* We will reset this sh_link in the reindex function. */ + sec->sh.sh_link = 0; + + relasec = sec->rela; + sym->sec->pfe = sec; + break; + } case PPC64: { bool found = false; @@ -3792,7 +3962,12 @@ static void kpatch_create_mcount_sections(struct kpatch_elf *kelf) mcount_rela->sym = sym; mcount_rela->type = absolute_rela_type(kelf); mcount_rela->addend = insn_offset - sym->sym.st_value; - mcount_rela->offset = (unsigned int) (index * sizeof(*funcs)); + + if (pfe_per_function) { + mcount_rela->offset = 0; + } else { + mcount_rela->offset = (unsigned int) (index * sizeof(*funcs)); + } index++; } @@ -3951,13 +4126,37 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) struct symbol *sym; struct rela *rela; unsigned char *insn; + list_for_each_entry(sym, &kelf->symbols, list) { - if (sym->type != STT_FUNC || sym->is_pfx || - !sym->sec || !sym->sec->rela) + if (sym->type != STT_FUNC || sym->is_pfx || !sym->sec) continue; switch(kelf->arch) { + case AARCH64: { + struct section *sec; + list_for_each_entry(sec, &kelf->sections, list) { + if (strcmp(sec->name, "__patchable_function_entries")) { + continue; + } + if (sym->sec->pfe != sec) { + continue; + } + if (!sec->rela) { + continue; + } + + list_for_each_entry(rela, &sec->rela->relas, list) { + if (rela->sym->sec && sym->sec == rela->sym->sec) { + sym->has_func_profiling = 1; + goto next_symbol; + } + } + } + break; + } case PPC64: + if (!sym->sec->rela) + continue; list_for_each_entry(rela, &sym->sec->rela->relas, list) { if (!strcmp(rela->sym->name, "_mcount")) { sym->has_func_profiling = 1; @@ -3966,6 +4165,8 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) } break; case X86_64: + if (!sym->sec->rela) + continue; rela = list_first_entry(&sym->sec->rela->relas, struct rela, list); if ((rela->type != R_X86_64_NONE && @@ -3977,6 +4178,8 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) sym->has_func_profiling = 1; break; case S390: + if (!sym->sec->rela) + continue; /* Check for compiler generated fentry nop - jgnop 0 */ insn = sym->sec->data->d_buf; if (insn[0] == 0xc0 && insn[1] == 0x04 && @@ -3987,6 +4190,7 @@ static void kpatch_find_func_profiling_calls(struct kpatch_elf *kelf) default: ERROR("unsupported arch"); } + next_symbol:; } } @@ -4067,6 +4271,10 @@ int main(int argc, char *argv[]) kelf_orig = kpatch_elf_open(orig_obj); kelf_patched = kpatch_elf_open(patched_obj); + + kpatch_set_pfe_link(kelf_orig); + kpatch_set_pfe_link(kelf_patched); + kpatch_find_func_profiling_calls(kelf_orig); kpatch_find_func_profiling_calls(kelf_patched); diff --git a/kpatch-build/create-kpatch-module.c b/kpatch-build/create-kpatch-module.c index 2884f93d9..b758bd4d9 100644 --- a/kpatch-build/create-kpatch-module.c +++ b/kpatch-build/create-kpatch-module.c @@ -58,6 +58,9 @@ static void create_dynamic_rela_sections(struct kpatch_elf *kelf, struct section dynsec = create_section_pair(kelf, ".kpatch.dynrelas", sizeof(*dynrelas), nr); dynrelas = dynsec->data->d_buf; + if (kelf->arch != X86_64) + return; + for (index = 0; index < nr; index++) { offset = index * (unsigned int)sizeof(*krelas); diff --git a/kpatch-build/kpatch-build b/kpatch-build/kpatch-build index ce5427994..82ecccdd7 100755 --- a/kpatch-build/kpatch-build +++ b/kpatch-build/kpatch-build @@ -81,7 +81,6 @@ declare -rA SUPPORTED_RPM_DISTROS=( ["rhel"]="RHEL" ["amzn"]="Amazon Linux") - warn() { echo "ERROR: $1" >&2 } @@ -379,6 +378,9 @@ find_special_section_data() { # Arch-specific features case "$ARCH" in + "aarch64") + check[a]=true # alt_instr + ;; "x86_64") check[a]=true # alt_instr kernel_version_gte 5.10.0 && check[s]=true # static_call_site diff --git a/kpatch-build/kpatch-cc b/kpatch-build/kpatch-cc index 17aae25b6..d5ec99362 100755 --- a/kpatch-build/kpatch-cc +++ b/kpatch-build/kpatch-cc @@ -42,6 +42,7 @@ if [[ "$TOOLCHAINCMD" =~ ^(.*-)?gcc$ || "$TOOLCHAINCMD" =~ ^(.*-)?clang$ ]] ; th arch/s390/boot/*|\ arch/s390/purgatory/*|\ arch/s390/kernel/vdso64/*|\ + arch/arm64/kernel/vdso*/*|\ drivers/firmware/efi/libstub/*|\ init/version.o|\ init/version-timestamp.o|\ diff --git a/kpatch-build/kpatch-elf.c b/kpatch-build/kpatch-elf.c old mode 100644 new mode 100755 index 374d424cc..a2223fc4a --- a/kpatch-build/kpatch-elf.c +++ b/kpatch-build/kpatch-elf.c @@ -38,6 +38,26 @@ * Helper functions ******************/ +static enum architecture current_arch; + +enum architecture def_arch(void) +{ + return current_arch; +} + +bool is_arch(enum architecture arch) +{ + return current_arch == arch; +} + +void set_arch(enum architecture arch) +{ + if (!arch || (current_arch && arch != current_arch)) + ERROR("inconsistent ELF arch: setting %d but already %d", + arch, current_arch); + current_arch = arch; +} + char *status_str(enum status status) { switch(status) { @@ -88,17 +108,29 @@ struct section *find_section_by_index(struct list_head *list, unsigned int index return NULL; } -struct section *find_section_by_name(struct list_head *list, const char *name) +struct section *find_nth_section_by_name( struct list_head *list, int nth, const char *name) { struct section *sec; - list_for_each_entry(sec, list, list) - if (!strcmp(sec->name, name)) - return sec; + if (!list || !list->next || !name) + return NULL; + + list_for_each_entry(sec, list, list) { + if (strcmp(sec->name, name)) + continue; + if (--nth >= 0) + continue; + return sec; + } return NULL; } +struct section *find_section_by_name(struct list_head *list, const char *name) +{ + return find_nth_section_by_name(list, 0, name); +} + struct symbol *find_symbol_by_index(struct list_head *list, size_t index) { struct symbol *sym; @@ -136,6 +168,8 @@ struct rela *find_rela_by_offset(struct section *relasec, unsigned int offset) unsigned int absolute_rela_type(struct kpatch_elf *kelf) { switch(kelf->arch) { + case AARCH64: + return R_AARCH64_ABS64; case PPC64: return R_PPC64_ADDR64; case X86_64: @@ -201,12 +235,12 @@ static void rela_insn(const struct section *sec, const struct rela *rela, long rela_target_offset(struct kpatch_elf *kelf, struct section *relasec, struct rela *rela) { - long add_off; + long add_off = 0; struct section *sec = relasec->base; switch(kelf->arch) { + case AARCH64: case PPC64: - add_off = 0; break; case X86_64: if (!is_text_section(sec) || @@ -254,6 +288,8 @@ unsigned int insn_length(struct kpatch_elf *kelf, void *addr) char *insn = addr; switch(kelf->arch) { + case AARCH64: + return 4; case X86_64: insn_init(&decoded_insn, addr, 1); @@ -584,6 +620,9 @@ struct kpatch_elf *kpatch_elf_open(const char *name) if (!gelf_getehdr(kelf->elf, &ehdr)) ERROR("gelf_getehdr"); switch (ehdr.e_machine) { + case EM_AARCH64: + kelf->arch = AARCH64; + break; case EM_PPC64: kelf->arch = PPC64; break; @@ -594,8 +633,10 @@ struct kpatch_elf *kpatch_elf_open(const char *name) kelf->arch = S390; break; default: - ERROR("Unsupported target architecture"); + ERROR("Unsupported target architecture: e_machine %x", + ehdr.e_machine); } + set_arch(kelf->arch); kpatch_create_section_list(kelf); kpatch_create_symbol_list(kelf); @@ -644,6 +685,7 @@ void kpatch_dump_kelf(struct kpatch_elf *kelf) if (sec->rela) printf(", rela-> %s", sec->rela->name); } + printf(", pfe-> [%d]", (sec->pfe) == NULL ? -1 : (int)sec->pfe->index); next: printf("\n"); } @@ -653,8 +695,10 @@ void kpatch_dump_kelf(struct kpatch_elf *kelf) printf("sym %02d, type %d, bind %d, ndx %02d, name %s (%s)", sym->index, sym->type, sym->bind, sym->sym.st_shndx, sym->name, status_str(sym->status)); - if (sym->sec && (sym->type == STT_FUNC || sym->type == STT_OBJECT)) + if (sym->sec && (sym->type == STT_FUNC || sym->type == STT_OBJECT)) { printf(" -> %s", sym->sec->name); + printf(", profiling: %d", sym->has_func_profiling); + } printf("\n"); } } @@ -923,6 +967,7 @@ struct section *create_section_pair(struct kpatch_elf *kelf, char *name, relasec->sh.sh_type = SHT_RELA; relasec->sh.sh_entsize = sizeof(GElf_Rela); relasec->sh.sh_addralign = 8; + relasec->sh.sh_flags = SHF_INFO_LINK; /* set text rela section pointer */ sec->rela = relasec; @@ -977,11 +1022,17 @@ void kpatch_reindex_elements(struct kpatch_elf *kelf) index = 0; list_for_each_entry(sym, &kelf->symbols, list) { sym->index = index++; - if (sym->sec) + if (sym->sec) { sym->sym.st_shndx = (unsigned short)sym->sec->index; - else if (sym->sym.st_shndx != SHN_ABS && - sym->sym.st_shndx != SHN_LIVEPATCH) + if (sym->sec->pfe) { + sym->sec->pfe->sh.sh_link = sym->sec->index; + if (sym->sec->pfe->rela) + sym->sec->pfe->rela->sh.sh_info = sym->sec->index; + } + } else if (sym->sym.st_shndx != SHN_ABS && + sym->sym.st_shndx != SHN_LIVEPATCH) { sym->sym.st_shndx = SHN_UNDEF; + } } } diff --git a/kpatch-build/kpatch-elf.h b/kpatch-build/kpatch-elf.h index e32209b72..64ebf8863 100644 --- a/kpatch-build/kpatch-elf.h +++ b/kpatch-build/kpatch-elf.h @@ -65,6 +65,7 @@ struct section { struct symbol *secsym, *sym; }; }; + struct section *pfe; /* arm64 per-func __patchable_function_entries */ }; enum symbol_strip { @@ -115,6 +116,7 @@ enum architecture { PPC64 = 0x1 << 0, X86_64 = 0x1 << 1, S390 = 0x1 << 2, + AARCH64 = 0x1 << 3, }; struct kpatch_elf { @@ -137,6 +139,8 @@ bool is_debug_section(struct section *sec); struct section *find_section_by_index(struct list_head *list, unsigned int index); struct section *find_section_by_name(struct list_head *list, const char *name); +struct section *find_nth_section_by_name(struct list_head *list, int nth, + const char *name); struct symbol *find_symbol_by_index(struct list_head *list, size_t index); struct symbol *find_symbol_by_name(struct list_head *list, const char *name); struct rela *find_rela_by_offset(struct section *relasec, unsigned int offset); @@ -157,6 +161,9 @@ int offset_of_string(struct list_head *list, char *name); long rela_target_offset(struct kpatch_elf *kelf, struct section *relasec, struct rela *rela); unsigned int insn_length(struct kpatch_elf *kelf, void *addr); +enum architecture def_arch(void); +void set_arch(enum architecture); +bool is_arch(enum architecture); #ifndef R_PPC64_ENTRY #define R_PPC64_ENTRY 118 diff --git a/test/unit/Makefile b/test/unit/Makefile index fde1717dd..e3ed7d718 100644 --- a/test/unit/Makefile +++ b/test/unit/Makefile @@ -1,4 +1,4 @@ -ARCHES ?= ppc64le x86_64 +ARCHES ?= aarch64 ppc64le x86_64 .PHONY: all clean submodule-check diff --git a/test/unit/objs b/test/unit/objs index a51c80a60..31f16a29c 160000 --- a/test/unit/objs +++ b/test/unit/objs @@ -1 +1 @@ -Subproject commit a51c80a60fc8ade7e7ec8ad875b2963f3a15a494 +Subproject commit 31f16a29c6c3dc9ac101d8ca780723a6667c219e