From 19fab7422441d77accc3e8802c28d43b29d978ae Mon Sep 17 00:00:00 2001 From: Tristan Ross Date: Thu, 26 Dec 2024 17:46:39 -0800 Subject: [PATCH] [libc] add basic arena allocator --- .github/workflows/libc-fullbuild-tests.yml | 1 + .github/workflows/libc-overlay-tests.yml | 1 + libc/CMakeLists.txt | 6 ++ libc/config/config.json | 4 ++ libc/docs/configure.rst | 1 + libc/src/__support/CMakeLists.txt | 2 + libc/src/__support/alloc/CMakeLists.txt | 56 +++++++++++++++ libc/src/__support/alloc/alloc.cpp | 13 ++++ libc/src/__support/alloc/alloc.h | 22 ++++++ libc/src/__support/alloc/arena.cpp | 71 +++++++++++++++++++ libc/src/__support/alloc/arena.h | 47 ++++++++++++ .../__support/alloc/baremetal/CMakeLists.txt | 13 ++++ libc/src/__support/alloc/baremetal/page.cpp | 22 ++++++ libc/src/__support/alloc/base.cpp | 16 +++++ libc/src/__support/alloc/base.h | 44 ++++++++++++ libc/src/__support/alloc/linux/CMakeLists.txt | 16 +++++ libc/src/__support/alloc/linux/page.cpp | 41 +++++++++++ libc/src/__support/alloc/page.h | 23 ++++++ libc/src/__support/memory_size.h | 14 ++++ libc/src/stdlib/CMakeLists.txt | 57 ++++++++++++++- libc/src/stdlib/aligned_alloc.cpp | 12 ++++ libc/src/stdlib/calloc.cpp | 12 ++++ libc/src/stdlib/free.cpp | 13 ++++ libc/src/stdlib/malloc.cpp | 13 ++++ libc/src/stdlib/realloc.cpp | 12 ++++ 25 files changed, 529 insertions(+), 3 deletions(-) create mode 100644 libc/src/__support/alloc/CMakeLists.txt create mode 100644 libc/src/__support/alloc/alloc.cpp create mode 100644 libc/src/__support/alloc/alloc.h create mode 100644 libc/src/__support/alloc/arena.cpp create mode 100644 libc/src/__support/alloc/arena.h create mode 100644 libc/src/__support/alloc/baremetal/CMakeLists.txt create mode 100644 libc/src/__support/alloc/baremetal/page.cpp create mode 100644 libc/src/__support/alloc/base.cpp create mode 100644 libc/src/__support/alloc/base.h create mode 100644 libc/src/__support/alloc/linux/CMakeLists.txt create mode 100644 libc/src/__support/alloc/linux/page.cpp create mode 100644 libc/src/__support/alloc/page.h create mode 100644 libc/src/stdlib/aligned_alloc.cpp create mode 100644 libc/src/stdlib/calloc.cpp create mode 100644 libc/src/stdlib/free.cpp create mode 100644 libc/src/stdlib/malloc.cpp create mode 100644 libc/src/stdlib/realloc.cpp diff --git a/.github/workflows/libc-fullbuild-tests.yml b/.github/workflows/libc-fullbuild-tests.yml index 58e15ce29546ef2..0893e7e65d7a66b 100644 --- a/.github/workflows/libc-fullbuild-tests.yml +++ b/.github/workflows/libc-fullbuild-tests.yml @@ -65,6 +65,7 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DCMAKE_INSTALL_PREFIX=${{ steps.strings.outputs.build-install-dir }} + -DLIBC_CONF_ALLOC_TYPE=LIBC_ALLOC_TYPE_SCUDO -DLLVM_ENABLE_RUNTIMES="libc;compiler-rt" -DLLVM_LIBC_FULL_BUILD=ON -DLLVM_LIBC_INCLUDE_SCUDO=ON diff --git a/.github/workflows/libc-overlay-tests.yml b/.github/workflows/libc-overlay-tests.yml index 8b59d76aed4a88a..b069040761850f4 100644 --- a/.github/workflows/libc-overlay-tests.yml +++ b/.github/workflows/libc-overlay-tests.yml @@ -87,6 +87,7 @@ jobs: -DCMAKE_POLICY_DEFAULT_CMP0141=NEW -DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded -DLLVM_ENABLE_RUNTIMES=libc + -DLIBC_CONF_ALLOC_TYPE=LIBC_ALLOC_TYPE_EXTERN -G Ninja -S ${{ github.workspace }}/runtimes diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt index 00a07ea3c8ac752..eebab925ba489d8 100644 --- a/libc/CMakeLists.txt +++ b/libc/CMakeLists.txt @@ -386,6 +386,12 @@ else() set(libc_opt_high_flag "-O3") endif() +if(${LIBC_CONF_ALLOC_TYPE} MATCHES "LIBC_ALLOC_TYPE_SCUDO") + set(LLVM_LIBC_INCLUDE_SCUDO ON) +elseif(LLVM_LIBC_INCLUDE_SCUDO) + message(FATAL_ERROR "Cannot include scudo and use a different allocator.") +endif() + add_subdirectory(include) add_subdirectory(config) add_subdirectory(hdr) diff --git a/libc/config/config.json b/libc/config/config.json index 23c057570d6fd79..6017d2e004de3e5 100644 --- a/libc/config/config.json +++ b/libc/config/config.json @@ -115,6 +115,10 @@ "LIBC_ADD_NULL_CHECKS": { "value": true, "doc": "Add nullptr checks in the library's implementations to some functions for which passing nullptr is undefined behavior." + }, + "LIBC_CONF_ALLOC_TYPE": { + "value": "LIBC_ALLOC_TYPE_ARENA", + "doc": "The implementation used for allocations, acceptable values are LIBC_ALLOC_TYPE_EXTERN, LIBC_ALLOC_TYPE_SCUDO, LIBC_ALLOC_TYPE_ARENA." } }, "unistd": { diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst index 89bfd93b8a99c2a..036e8fc7a938328 100644 --- a/libc/docs/configure.rst +++ b/libc/docs/configure.rst @@ -32,6 +32,7 @@ to learn about the defaults for your platform and target. - ``LIBC_CONF_ERRNO_MODE``: The implementation used for errno, acceptable values are LIBC_ERRNO_MODE_DEFAULT, LIBC_ERRNO_MODE_UNDEFINED, LIBC_ERRNO_MODE_THREAD_LOCAL, LIBC_ERRNO_MODE_SHARED, LIBC_ERRNO_MODE_EXTERNAL, and LIBC_ERRNO_MODE_SYSTEM. * **"general" options** - ``LIBC_ADD_NULL_CHECKS``: Add nullptr checks in the library's implementations to some functions for which passing nullptr is undefined behavior. + - ``LIBC_CONF_ALLOC_TYPE``: The implementation used for allocations, acceptable values are LIBC_ALLOC_TYPE_EXTERN, LIBC_ALLOC_TYPE_SCUDO, LIBC_ALLOC_TYPE_ARENA. * **"math" options** - ``LIBC_CONF_FREXP_INF_NAN_EXPONENT``: The value written back to the second parameter when calling frexp/frexpf/frexpl` with `+/-Inf`/`NaN` is unspecified. Configue an explicit exp value for Inf/NaN inputs. - ``LIBC_CONF_MATH_OPTIMIZATIONS``: Configures optimizations for math functions. Values accepted are LIBC_MATH_SKIP_ACCURATE_PASS, LIBC_MATH_SMALL_TABLES, LIBC_MATH_NO_ERRNO, LIBC_MATH_NO_EXCEPT, and LIBC_MATH_FAST. diff --git a/libc/src/__support/CMakeLists.txt b/libc/src/__support/CMakeLists.txt index 4e90aad9a45b40c..45f4dde264122ab 100644 --- a/libc/src/__support/CMakeLists.txt +++ b/libc/src/__support/CMakeLists.txt @@ -370,3 +370,5 @@ add_subdirectory(HashTable) add_subdirectory(fixed_point) add_subdirectory(time) + +add_subdirectory(alloc) diff --git a/libc/src/__support/alloc/CMakeLists.txt b/libc/src/__support/alloc/CMakeLists.txt new file mode 100644 index 000000000000000..38571d0300dff7f --- /dev/null +++ b/libc/src/__support/alloc/CMakeLists.txt @@ -0,0 +1,56 @@ +add_object_library( + base + SRCS + base.cpp + HDRS + base.h + DEPENDS + libc.hdr.types.size_t + libc.src.__support.macros.config +) + +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) + add_subdirectory(${LIBC_TARGET_OS}) +endif() + +if(TARGET libc.src.__support.alloc.${LIBC_TARGET_OS}.page) + add_object_library( + page + ALIAS + DEPENDS + .${LIBC_TARGET_OS}.page + ) +endif() + +add_object_library( + arena + SRCS + arena.cpp + HDRS + arena.h + COMPILE_OPTIONS + -DLIBC_PAGE_SIZE=${LIBC_CONF_PAGE_SIZE} + DEPENDS + .base + .page + libc.src.string.memmove + libc.src.unistd.getpagesize +) + +if(NOT ${LIBC_CONF_ALLOC_TYPE} MATCHES "LIBC_ALLOC_TYPE_SCUDO" AND NOT ${LIBC_CONF_ALLOC_TYPE} MATCHES "LIBC_ALLOC_TYPE_EXTERN") + string(TOLOWER ${LIBC_CONF_ALLOC_TYPE} LIBC_CONF_ALLOC_TYPE_NAME) + string(REPLACE "libc_alloc_type_" "" LIBC_CONF_ALLOC_TYPE_NAME "${LIBC_CONF_ALLOC_TYPE_NAME}") + if(TARGET libc.src.__support.alloc.${LIBC_CONF_ALLOC_TYPE_NAME}) + add_object_library( + alloc + SRCS + alloc.cpp + HDRS + alloc.h + COMPILE_OPTIONS + -DLIBC_CONF_ALLOC_TYPE=${LIBC_CONF_ALLOC_TYPE_NAME} + DEPENDS + .${LIBC_CONF_ALLOC_TYPE_NAME} + ) + endif() +endif() diff --git a/libc/src/__support/alloc/alloc.cpp b/libc/src/__support/alloc/alloc.cpp new file mode 100644 index 000000000000000..f5f1832bb58a8f4 --- /dev/null +++ b/libc/src/__support/alloc/alloc.cpp @@ -0,0 +1,13 @@ +#include +#include + +namespace LIBC_NAMESPACE_DECL { + +#define CONCAT(a, b) a##b +#define EXPAND_AND_CONCAT(a, b) CONCAT(a, b) + +#define ALLOCATOR EXPAND_AND_CONCAT(LIBC_CONF_ALLOC_TYPE, _allocator) + +BaseAllocator *allocator = reinterpret_cast(&ALLOCATOR); + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/alloc/alloc.h b/libc/src/__support/alloc/alloc.h new file mode 100644 index 000000000000000..88b5fbfb84680f4 --- /dev/null +++ b/libc/src/__support/alloc/alloc.h @@ -0,0 +1,22 @@ +//===-- libc-wide allocator -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_ALLOC_ALLOC_H +#define LLVM_LIBC_SRC___SUPPORT_ALLOC_ALLOC_H + +#include "src/__support/alloc/base.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +// The primary allocator to use +extern BaseAllocator *allocator; + +} // namespace LIBC_NAMESPACE_DECL + +#endif diff --git a/libc/src/__support/alloc/arena.cpp b/libc/src/__support/alloc/arena.cpp new file mode 100644 index 000000000000000..f39d80630f24774 --- /dev/null +++ b/libc/src/__support/alloc/arena.cpp @@ -0,0 +1,71 @@ +#include "src/__support/alloc/arena.h" +#include "src/__support/alloc/page.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/__support/macros/page_size.h" +#include "src/__support/memory_size.h" +#include "src/string/memmove.h" +#include "src/unistd/getpagesize.h" + +namespace LIBC_NAMESPACE_DECL { + +void *arena_allocate(BaseAllocator *base, size_t alignment, size_t size) { + ArenaAllocator *self = reinterpret_cast(base); + + if (self->buffer == nullptr) { + self->buffer = reinterpret_cast(page_allocate(1)); + self->buffer_size = self->get_page_size(); + } + + uintptr_t curr_ptr = (uintptr_t)self->buffer + (uintptr_t)self->curr_offset; + uintptr_t offset = internal::align_forward(curr_ptr, alignment); + offset -= (uintptr_t)self->buffer; + + if (offset + size > self->buffer_size) { + self->buffer = reinterpret_cast( + page_expand(self->buffer, self->buffer_size / self->get_page_size())); + self->buffer_size += self->get_page_size(); + } + + if (offset + size <= self->buffer_size) { + void *ptr = &self->buffer[offset]; + self->prev_offset = offset; + self->curr_offset = offset + size; + return ptr; + } + return nullptr; +} + +void *arena_expand(BaseAllocator *base, void *ptr, size_t alignment, + size_t size) { + ArenaAllocator *self = reinterpret_cast(base); + + if (self->buffer + self->prev_offset == ptr) { + self->curr_offset = self->prev_offset + size; + return ptr; + } else { + void *new_mem = arena_allocate(base, alignment, size); + memmove(new_mem, ptr, size); + return new_mem; + } + return nullptr; +} + +bool arena_free(BaseAllocator *base, void *ptr) { + (void)base; + (void)ptr; + return true; +} + +size_t ArenaAllocator::get_page_size() { + if (page_size == LIBC_PAGE_SIZE_SYSTEM) { + page_size = getpagesize(); + } + return page_size; +} + +static ArenaAllocator default_arena_allocator(LIBC_PAGE_SIZE, + 2 * sizeof(void *)); +BaseAllocator *block_allocator = &default_arena_allocator; + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/alloc/arena.h b/libc/src/__support/alloc/arena.h new file mode 100644 index 000000000000000..a0af745bc7410d3 --- /dev/null +++ b/libc/src/__support/alloc/arena.h @@ -0,0 +1,47 @@ +//===-- An arena allocator using pages. -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_ALLOC_ARENA_H +#define LLVM_LIBC_SRC___SUPPORT_ALLOC_ARENA_H + +#include "hdr/types/size_t.h" +#include "src/__support/alloc/base.h" +#include + +namespace LIBC_NAMESPACE_DECL { + +void *arena_allocate(BaseAllocator *base, size_t alignment, size_t size); +void *arena_expand(BaseAllocator *base, void *ptr, size_t alignment, + size_t size); +bool arena_free(BaseAllocator *base, void *ptr); + +class ArenaAllocator : public BaseAllocator { +public: + uint8_t *buffer; + size_t buffer_size; + size_t prev_offset; + size_t curr_offset; + +private: + size_t page_size; + +public: + constexpr ArenaAllocator(size_t page_size, size_t default_alignment) + : BaseAllocator(arena_allocate, arena_expand, arena_free, + default_alignment), + buffer(nullptr), buffer_size(0), prev_offset(0), curr_offset(0), + page_size(page_size) {} + + size_t get_page_size(); +}; + +extern BaseAllocator *arena_allocator; + +} // namespace LIBC_NAMESPACE_DECL + +#endif diff --git a/libc/src/__support/alloc/baremetal/CMakeLists.txt b/libc/src/__support/alloc/baremetal/CMakeLists.txt new file mode 100644 index 000000000000000..845d2f5287bfbf1 --- /dev/null +++ b/libc/src/__support/alloc/baremetal/CMakeLists.txt @@ -0,0 +1,13 @@ +add_object_library( + page + SRCS + page.cpp + HDRS + ../page.h + DEPENDS + libc.src.__support.alloc.base + libc.src.sys.mman.mmap + libc.src.sys.mman.mremap + libc.src.sys.mman.munmap + libc.src.unistd.getpagesize +) diff --git a/libc/src/__support/alloc/baremetal/page.cpp b/libc/src/__support/alloc/baremetal/page.cpp new file mode 100644 index 000000000000000..4d92141bcc82764 --- /dev/null +++ b/libc/src/__support/alloc/baremetal/page.cpp @@ -0,0 +1,22 @@ +#include "src/__support/alloc/page.h" +#include "src/__suport/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +extern "C" void *__llvm_libc_page_allocate(size_t n_pages); +extern "C" void *__llvm_libc_page_expand(void *ptr, size_t n_pages); +extern "C" bool __llvm_libc_page_free(void *ptr, size_t n_pages); + +void *page_allocate(size_t n_pages) { + return __llvm_libc_page_allocate(n_pages); +} + +void *page_expand(void *ptr, size_t n_pages) { + return __llvm_libc_page_expand(ptr, n_pages); +} + +bool page_free(void *ptr, size_t n_pages) { + return __llvm_libc_page_free(ptr, n_pages); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/alloc/base.cpp b/libc/src/__support/alloc/base.cpp new file mode 100644 index 000000000000000..93684d457ee42d0 --- /dev/null +++ b/libc/src/__support/alloc/base.cpp @@ -0,0 +1,16 @@ +#include "src/__support/alloc/base.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +void *BaseAllocator::alloc(size_t alignment, size_t size) { + return impl_alloc(this, alignment, size); +} + +void *BaseAllocator::expand(void *ptr, size_t alignment, size_t size) { + return impl_expand(this, ptr, alignment, size); +} + +bool BaseAllocator::free(void *ptr) { return impl_free(this, ptr); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/alloc/base.h b/libc/src/__support/alloc/base.h new file mode 100644 index 000000000000000..59370352953d3e5 --- /dev/null +++ b/libc/src/__support/alloc/base.h @@ -0,0 +1,44 @@ +//===-- A generic base allocator. -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_ALLOC_BASE_H +#define LLVM_LIBC_SRC___SUPPORT_ALLOC_BASE_H + +#include "hdr/types/size_t.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +class BaseAllocator { +public: + using AllocFunc = void *(BaseAllocator *self, size_t, size_t); + using ExpandFunc = void *(BaseAllocator *self, void *, size_t, size_t); + using FreeFunc = bool(BaseAllocator *self, void *); + +private: + // Implementation specific functions + AllocFunc *impl_alloc; + ExpandFunc *impl_expand; + FreeFunc *impl_free; + +public: + constexpr BaseAllocator(AllocFunc *ia, ExpandFunc *ie, FreeFunc *ifr, + size_t default_alignment) + : impl_alloc(ia), impl_expand(ie), impl_free(ifr), + default_alignment(default_alignment) {} + + size_t default_alignment; + + void *alloc(size_t alignment, size_t size); + void *expand(void *ptr, size_t alignment, size_t size); + bool free(void *ptr); +}; + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_ALLOC_BASE_H diff --git a/libc/src/__support/alloc/linux/CMakeLists.txt b/libc/src/__support/alloc/linux/CMakeLists.txt new file mode 100644 index 000000000000000..4484a5d60c96350 --- /dev/null +++ b/libc/src/__support/alloc/linux/CMakeLists.txt @@ -0,0 +1,16 @@ +add_object_library( + page + SRCS + page.cpp + HDRS + ../page.h + DEPENDS + libc.hdr.types.size_t + libc.include.llvm-libc-macros.stdlib_macros + libc.include.llvm-libc-macros.stdint_macros + libc.src.__support.alloc.base + libc.src.sys.mman.mmap + libc.src.sys.mman.mremap + libc.src.sys.mman.munmap + libc.src.unistd.getpagesize +) diff --git a/libc/src/__support/alloc/linux/page.cpp b/libc/src/__support/alloc/linux/page.cpp new file mode 100644 index 000000000000000..739d1f69915ba50 --- /dev/null +++ b/libc/src/__support/alloc/linux/page.cpp @@ -0,0 +1,41 @@ +#include "src/__support/alloc/page.h" +#include "include/llvm-libc-macros/stdlib-macros.h" +#include "src/__support/macros/config.h" +#include "src/__support/memory_size.h" +#include "src/sys/mman/mmap.h" +#include "src/sys/mman/mremap.h" +#include "src/sys/mman/munmap.h" +#include "src/unistd/getpagesize.h" +#include + +namespace LIBC_NAMESPACE_DECL { + +static void *alloc_hint = NULL; + +void *page_allocate(size_t n_pages) { + size_t page_size = getpagesize(); + size_t size = n_pages * page_size; + size_t aligned_size = internal::SafeMemSize(size).align_up(page_size); + + void *ptr = mmap(&alloc_hint, aligned_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == NULL) + return nullptr; + + alloc_hint = (void *)(((uintptr_t)ptr) + aligned_size); + return ptr; +} + +void *page_expand(void *ptr, size_t n_pages) { + (void)ptr; + (void)n_pages; + return nullptr; +} + +bool page_free(void *ptr, size_t n_pages) { + (void)ptr; + (void)n_pages; + return false; +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/alloc/page.h b/libc/src/__support/alloc/page.h new file mode 100644 index 000000000000000..bb627b50cd6eb4c --- /dev/null +++ b/libc/src/__support/alloc/page.h @@ -0,0 +1,23 @@ +//===-- Page allocations ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_ALLOC_PAGE_H +#define LLVM_LIBC_SRC___SUPPORT_ALLOC_PAGE_H + +#include "hdr/types/size_t.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +void *page_allocate(size_t n_pages); +void *page_expand(void *ptr, size_t n_pages); +bool page_free(void *ptr, size_t n_pages); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_ALLOC_BASE_H diff --git a/libc/src/__support/memory_size.h b/libc/src/__support/memory_size.h index cdd6a10222de105..a9df814be7156b6 100644 --- a/libc/src/__support/memory_size.h +++ b/libc/src/__support/memory_size.h @@ -12,6 +12,7 @@ #include "src/__support/CPP/bit.h" // has_single_bit #include "src/__support/CPP/limits.h" #include "src/__support/CPP/type_traits.h" +#include "src/__support/libc_assert.h" #include "src/__support/macros/attributes.h" #include "src/__support/macros/config.h" #include "src/__support/macros/optimization.h" @@ -32,6 +33,19 @@ template LIBC_INLINE bool mul_overflow(T a, T b, T *res) { return overflow; #endif } + +template LIBC_INLINE T align_forward(T ptr, size_t align) { + LIBC_ASSERT((align & (align - 1)) == 0); + + T p = ptr; + T a = (T)align; + + T modulo = p & (a - 1); + if (modulo != 0) + p += a - modulo; + return p; +} + // Limit memory size to the max of ssize_t class SafeMemSize { private: diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt index 40ba9ead9a7ae6a..4e582df6d301f7a 100644 --- a/libc/src/stdlib/CMakeLists.txt +++ b/libc/src/stdlib/CMakeLists.txt @@ -324,7 +324,7 @@ add_entrypoint_object( ) if(NOT LIBC_TARGET_OS_IS_BAREMETAL AND NOT LIBC_TARGET_OS_IS_GPU) - if(LLVM_LIBC_INCLUDE_SCUDO) + if(LLVM_LIBC_INCLUDE_SCUDO AND ${LIBC_CONF_ALLOC_TYPE} MATCHES "LIBC_CONF_ALLOC_TYPE_SCUDO") set(SCUDO_DEPS "") include(${LIBC_SOURCE_DIR}/../compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake) @@ -388,7 +388,7 @@ if(NOT LIBC_TARGET_OS_IS_BAREMETAL AND NOT LIBC_TARGET_OS_IS_GPU) DEPENDS ${SCUDO_DEPS} ) - else() + elseif(${LIBC_CONF_ALLOC_TYPE} MATCHES "LIBC_CONF_ALLOC_TYPE_EXTERN") add_entrypoint_external( malloc ) @@ -407,6 +407,57 @@ if(NOT LIBC_TARGET_OS_IS_BAREMETAL AND NOT LIBC_TARGET_OS_IS_GPU) add_entrypoint_external( mallopt ) + else(TARGET libc.src.__support.alloc.alloc) + add_entrypoint_object( + aligned_alloc + SRCS + aligned_alloc.cpp + HDRS + aligned_alloc.h + DEPENDS + libc.src.__support.alloc.alloc + ) + + add_entrypoint_object( + malloc + SRCS + malloc.cpp + HDRS + malloc.h + DEPENDS + .aligned_alloc + ) + + add_entrypoint_object( + calloc + SRCS + calloc.cpp + HDRS + calloc.h + DEPENDS + .malloc + ) + + add_entrypoint_object( + realloc + SRCS + realloc.cpp + HDRS + realloc.h + DEPENDS + libc.src.__support.alloc.alloc + ) + + add_entrypoint_object( + free + SRCS + free.cpp + HDRS + free.h + DEPENDS + libc.src.__support.libc_assert + libc.src.__support.alloc.alloc + ) endif() endif() @@ -495,7 +546,7 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) endif() -if(LIBC_TARGET_OS_IS_BAREMETAL OR LIBC_TARGET_OS_IS_GPU) +if((LIBC_TARGET_OS_IS_BAREMETAL OR LIBC_TARGET_OS_IS_GPU) AND NOT TARGET libc.src.__support.alloc.alloc) add_entrypoint_object( malloc ALIAS diff --git a/libc/src/stdlib/aligned_alloc.cpp b/libc/src/stdlib/aligned_alloc.cpp new file mode 100644 index 000000000000000..d7883363486f8b7 --- /dev/null +++ b/libc/src/stdlib/aligned_alloc.cpp @@ -0,0 +1,12 @@ +#include "src/stdlib/aligned_alloc.h" +#include "src/__support/alloc/alloc.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(void *, aligned_alloc, (size_t alignment, size_t size)) { + return allocator->alloc(alignment, size); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/stdlib/calloc.cpp b/libc/src/stdlib/calloc.cpp new file mode 100644 index 000000000000000..d2bebd2eb7d7a14 --- /dev/null +++ b/libc/src/stdlib/calloc.cpp @@ -0,0 +1,12 @@ +#include "src/stdlib/calloc.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/stdlib/malloc.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(void *, calloc, (size_t nmeb, size_t size)) { + return malloc(nmeb * size); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/stdlib/free.cpp b/libc/src/stdlib/free.cpp new file mode 100644 index 000000000000000..71a77e615cf2d13 --- /dev/null +++ b/libc/src/stdlib/free.cpp @@ -0,0 +1,13 @@ +#include "src/stdlib/free.h" +#include "src/__support/alloc/alloc.h" +#include "src/__support/common.h" +#include "src/__support/libc_assert.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(void, free, (void *ptr)) { + LIBC_ASSERT(allocator->free(ptr)); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/stdlib/malloc.cpp b/libc/src/stdlib/malloc.cpp new file mode 100644 index 000000000000000..a8a51d10ebb7326 --- /dev/null +++ b/libc/src/stdlib/malloc.cpp @@ -0,0 +1,13 @@ +#include "src/stdlib/malloc.h" +#include "src/__support/alloc/alloc.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" +#include "src/stdlib/aligned_alloc.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(void *, malloc, (size_t size)) { + return aligned_alloc(allocator->default_alignment, size); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/stdlib/realloc.cpp b/libc/src/stdlib/realloc.cpp new file mode 100644 index 000000000000000..60043bf8ad287a3 --- /dev/null +++ b/libc/src/stdlib/realloc.cpp @@ -0,0 +1,12 @@ +#include "src/stdlib/realloc.h" +#include "src/__support/alloc/alloc.h" +#include "src/__support/common.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(void *, realloc, (void *ptr, size_t size)) { + return allocator->expand(ptr, allocator->default_alignment, size); +} + +} // namespace LIBC_NAMESPACE_DECL