diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index e5f881bc8288..6903700afd12 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -1223,6 +1223,8 @@ arch_get_unmapped_area_1(unsigned long addr, unsigned long len, info.length = len; info.low_limit = addr; info.high_limit = limit; + info.hint = addr; + info.mmap_flags = flags; return vm_unmapped_area(&info); } diff --git a/arch/arc/mm/mmap.c b/arch/arc/mm/mmap.c index 69a915297155..5922cb51e029 100644 --- a/arch/arc/mm/mmap.c +++ b/arch/arc/mm/mmap.c @@ -29,6 +29,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, struct vm_area_struct *vma; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + /* * We enforce the MAP_FIXED case. */ diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index d65d0e6ed10a..04d9234f049a 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -36,6 +36,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, int aliasing = cache_is_vipt_aliasing(); struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + /* * We only need to do colour alignment if either the I or D * caches alias. @@ -56,6 +59,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, if (len > TASK_SIZE) return -ENOMEM; + if (addr) { if (do_align) addr = COLOUR_ALIGN(addr, pgoff); @@ -88,6 +92,9 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, int aliasing = cache_is_vipt_aliasing(); struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + /* * We only need to do colour alignment if either the I or D * caches alias. diff --git a/arch/csky/abiv1/mmap.c b/arch/csky/abiv1/mmap.c index 7f826331d409..0be4913c6cf3 100644 --- a/arch/csky/abiv1/mmap.c +++ b/arch/csky/abiv1/mmap.c @@ -35,6 +35,9 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, .align_offset = pgoff << PAGE_SHIFT }; + info.hint = addr; + info.mmap_flags = flags; + /* * We only need to do colour alignment if either the I or D * caches alias. diff --git a/arch/loongarch/mm/mmap.c b/arch/loongarch/mm/mmap.c index 889030985135..7d1e8be20519 100644 --- a/arch/loongarch/mm/mmap.c +++ b/arch/loongarch/mm/mmap.c @@ -27,6 +27,9 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, int do_color_align; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + if (unlikely(len > TASK_SIZE)) return -ENOMEM; diff --git a/arch/mips/mm/mmap.c b/arch/mips/mm/mmap.c index 7e11d7b58761..22e8f9c8eaa0 100644 --- a/arch/mips/mm/mmap.c +++ b/arch/mips/mm/mmap.c @@ -36,6 +36,9 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, int do_color_align; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + if (unlikely(len > TASK_SIZE)) return -ENOMEM; diff --git a/arch/parisc/kernel/sys_parisc.c b/arch/parisc/kernel/sys_parisc.c index f7722451276e..2ac53f148624 100644 --- a/arch/parisc/kernel/sys_parisc.c +++ b/arch/parisc/kernel/sys_parisc.c @@ -108,6 +108,9 @@ static unsigned long arch_get_unmapped_area_common(struct file *filp, .length = len }; + info.hint = addr; + info.mmap_flags = flags; + if (unlikely(len > TASK_SIZE)) return -ENOMEM; diff --git a/arch/powerpc/mm/book3s64/slice.c b/arch/powerpc/mm/book3s64/slice.c index ef3ce37f1bb3..f0e2550af6d0 100644 --- a/arch/powerpc/mm/book3s64/slice.c +++ b/arch/powerpc/mm/book3s64/slice.c @@ -286,6 +286,10 @@ static unsigned long slice_find_area_bottomup(struct mm_struct *mm, .length = len, .align_mask = PAGE_MASK & ((1ul << pshift) - 1), }; + + info.hint = addr; + info.mmap_flags = flags; + /* * Check till the allow max value for this mmap request */ @@ -331,6 +335,9 @@ static unsigned long slice_find_area_topdown(struct mm_struct *mm, }; unsigned long min_addr = max(PAGE_SIZE, mmap_min_addr); + info.hint = addr; + info.mmap_flags = flags; + /* * If we are trying to allocate above DEFAULT_MAP_WINDOW * Add the different to the mmap_base. diff --git a/arch/s390/mm/hugetlbpage.c b/arch/s390/mm/hugetlbpage.c index ded0eff58a19..dc38f7022037 100644 --- a/arch/s390/mm/hugetlbpage.c +++ b/arch/s390/mm/hugetlbpage.c @@ -254,6 +254,8 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, info.low_limit = current->mm->mmap_base; info.high_limit = TASK_SIZE; info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.hint = addr; + info.mmap_flags = flags; return vm_unmapped_area(&info); } @@ -270,6 +272,8 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, info.low_limit = PAGE_SIZE; info.high_limit = current->mm->mmap_base; info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.hint = addr; + info.mmap_flags = flags; addr = vm_unmapped_area(&info); /* diff --git a/arch/s390/mm/mmap.c b/arch/s390/mm/mmap.c index 206756946589..001345df61b6 100644 --- a/arch/s390/mm/mmap.c +++ b/arch/s390/mm/mmap.c @@ -88,6 +88,9 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, struct vm_area_struct *vma; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + if (len > TASK_SIZE - mmap_min_addr) return -ENOMEM; @@ -123,6 +126,9 @@ unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long ad struct mm_struct *mm = current->mm; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + /* requested length too big for entire address space */ if (len > TASK_SIZE - mmap_min_addr) return -ENOMEM; diff --git a/arch/sh/mm/mmap.c b/arch/sh/mm/mmap.c index bee329d4149a..ab1f5613b5e2 100644 --- a/arch/sh/mm/mmap.c +++ b/arch/sh/mm/mmap.c @@ -59,6 +59,9 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, int do_colour_align; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate * cache aliasing constraints. @@ -107,6 +110,9 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, int do_colour_align; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate * cache aliasing constraints. diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c index 08a19727795c..bcc3242e003d 100644 --- a/arch/sparc/kernel/sys_sparc_32.c +++ b/arch/sparc/kernel/sys_sparc_32.c @@ -43,6 +43,9 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi { struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate * cache aliasing constraints. diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index d9c3b34ca744..05de444d66f6 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -95,6 +95,9 @@ unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsi int do_color_align; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + if (flags & MAP_FIXED) { /* We do not accept a shared mapping if it would violate * cache aliasing constraints. @@ -155,6 +158,9 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, int do_color_align; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + /* This should only ever run for 32-bit processes. */ BUG_ON(!test_thread_flag(TIF_32BIT)); diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index cc91ca7a1e18..148064142628 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -40,6 +40,8 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *filp, info.low_limit = TASK_UNMAPPED_BASE; info.high_limit = min(task_size, VA_EXCLUDE_START); info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.hint = addr; + info.mmap_flags = flags; addr = vm_unmapped_area(&info); if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { @@ -71,6 +73,8 @@ hugetlb_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, info.low_limit = PAGE_SIZE; info.high_limit = mm->mmap_base; info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.hint = addr; + info.mmap_flags = flags; addr = vm_unmapped_area(&info); /* diff --git a/arch/x86/kernel/sys_x86_64.c b/arch/x86/kernel/sys_x86_64.c index 01d7cd85ef97..248a210c2b0b 100644 --- a/arch/x86/kernel/sys_x86_64.c +++ b/arch/x86/kernel/sys_x86_64.c @@ -129,6 +129,9 @@ arch_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned l struct vm_unmapped_area_info info = {}; unsigned long begin, end; + info.hint = addr; + info.mmap_flags = flags; + if (flags & MAP_FIXED) return addr; @@ -167,6 +170,9 @@ arch_get_unmapped_area_topdown_vmflags(struct file *filp, unsigned long addr0, unsigned long addr = addr0; struct vm_unmapped_area_info info = {}; + info.hint = addr; + info.mmap_flags = flags; + /* requested length too big for entire address space */ if (len > TASK_SIZE) return -ENOMEM; diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c index 807a5859a3c4..d3f8cec4895d 100644 --- a/arch/x86/mm/hugetlbpage.c +++ b/arch/x86/mm/hugetlbpage.c @@ -29,6 +29,8 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file, info.length = len; info.low_limit = get_mmap_base(1); + info.hint = addr; + info.mmap_flags = flags; /* * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area @@ -52,6 +54,8 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file, info.length = len; info.low_limit = PAGE_SIZE; info.high_limit = get_mmap_base(0); + info.hint = addr; + info.mmap_flags = flags; /* * If hint address is above DEFAULT_MAP_WINDOW, look for unmapped area diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 9f6cff356796..4b35eadd2ad8 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -182,6 +182,8 @@ hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long addr, info.low_limit = current->mm->mmap_base; info.high_limit = arch_get_mmap_end(addr, len, flags); info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.hint = addr; + info.mmap_flags = flags; return vm_unmapped_area(&info); } @@ -197,6 +199,8 @@ hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr, info.low_limit = PAGE_SIZE; info.high_limit = arch_get_mmap_base(addr, current->mm->mmap_base); info.align_mask = PAGE_MASK & ~huge_page_mask(h); + info.hint = addr; + info.mmap_flags = flags; addr = vm_unmapped_area(&info); /* diff --git a/include/linux/mm.h b/include/linux/mm.h index c4b238a20b76..3d24703d42de 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3434,6 +3434,8 @@ struct vm_unmapped_area_info { unsigned long align_mask; unsigned long align_offset; unsigned long start_gap; + unsigned long hint; + unsigned long mmap_flags; }; extern unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info); diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h index 6ce1f1ceb432..03ac13d9aa37 100644 --- a/include/uapi/asm-generic/mman-common.h +++ b/include/uapi/asm-generic/mman-common.h @@ -32,6 +32,7 @@ #define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be * uninitialized */ +#define MAP_BELOW_HINT 0x8000000 /* give out address that is below (inclusive) hint address */ /* * Flags for mlock diff --git a/mm/mmap.c b/mm/mmap.c index d0dfc85b209b..459ad380c673 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -1766,6 +1766,9 @@ unsigned long vm_unmapped_area(struct vm_unmapped_area_info *info) { unsigned long addr; + if (info->hint != 0 && info->mmap_flags & MAP_BELOW_HINT) + info->high_limit = MIN(info->high_limit, info->hint + info->length); + if (info->flags & VM_UNMAPPED_AREA_TOPDOWN) addr = unmapped_area_topdown(info); else @@ -1796,6 +1799,9 @@ generic_get_unmapped_area(struct file *filp, unsigned long addr, struct vm_unmapped_area_info info = {}; const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); + info.hint = addr; + info.mmap_flags = flags; + if (len > mmap_end - mmap_min_addr) return -ENOMEM; @@ -1841,6 +1847,9 @@ generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, struct vm_unmapped_area_info info = {}; const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags); + info.hint = addr; + info.mmap_flags = flags; + /* requested length too big for entire address space */ if (len > mmap_end - mmap_min_addr) return -ENOMEM; diff --git a/tools/include/uapi/asm-generic/mman-common.h b/tools/include/uapi/asm-generic/mman-common.h index 6ce1f1ceb432..03ac13d9aa37 100644 --- a/tools/include/uapi/asm-generic/mman-common.h +++ b/tools/include/uapi/asm-generic/mman-common.h @@ -32,6 +32,7 @@ #define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be * uninitialized */ +#define MAP_BELOW_HINT 0x8000000 /* give out address that is below (inclusive) hint address */ /* * Flags for mlock diff --git a/tools/testing/selftests/mm/Makefile b/tools/testing/selftests/mm/Makefile index 901e0d07765b..0c652ff49d4b 100644 --- a/tools/testing/selftests/mm/Makefile +++ b/tools/testing/selftests/mm/Makefile @@ -50,6 +50,7 @@ TEST_GEN_FILES += hugepage-shm TEST_GEN_FILES += hugepage-vmemmap TEST_GEN_FILES += khugepaged TEST_GEN_FILES += madv_populate +TEST_GEN_FILES += map_below_hint TEST_GEN_FILES += map_fixed_noreplace TEST_GEN_FILES += map_hugetlb TEST_GEN_FILES += map_populate diff --git a/tools/testing/selftests/mm/map_below_hint.c b/tools/testing/selftests/mm/map_below_hint.c new file mode 100644 index 000000000000..55d6cbf90645 --- /dev/null +++ b/tools/testing/selftests/mm/map_below_hint.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Test the MAP_BELOW_HINT mmap flag. + */ +#include +#include +#include "../kselftest.h" + +#define ADDR (1 << 20) +#define LENGTH (ADDR / 10000) + +#define MAP_BELOW_HINT 0x8000000 /* Not defined in all libc */ + +/* + * Map memory with MAP_BELOW_HINT until no memory left. Ensure that all returned + * addresses are below the hint. + */ +int main(int argc, char **argv) +{ + void *addr; + + do { + addr = mmap((void *)ADDR, LENGTH, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_BELOW_HINT, -1, 0); + } while (addr != MAP_FAILED && (unsigned long)addr <= ADDR); + + if (errno == ENOMEM) + ksft_test_result_pass("MAP_BELOW_HINT works\n"); + else + ksft_test_result_fail("mmap returned address above hint with MAP_BELOW_HINT with error: %s\n", + strerror(errno)); +}