diff --git a/include/svs/core/allocator.h b/include/svs/core/allocator.h index 1e449a7e..c22207fc 100644 --- a/include/svs/core/allocator.h +++ b/include/svs/core/allocator.h @@ -20,6 +20,11 @@ /// @file /// @brief Implements common large-scale allocators used by the many data structures. /// +/// Memory Population Control: +/// Setting environment variable `SVS_ENABLE_MAP_POPULATE` enables MAP_POPULATE for +/// hugepage and file-backed mappings (Linux only). By default population is disabled +/// to avoid transient RSS spikes during large index loads. Use the variable for +/// benchmarking scenarios where early page faulting is desired. /// /// @ingroup core @@ -112,6 +117,13 @@ struct HugepageAllocation { assert(bytes != 0); void* ptr = MAP_FAILED; size_t sz = 0; + // Allow disabling page population via environment variable to mitigate peak RSS. + // If SVS_DISABLE_MAP_POPULATE is set to a non-empty value, MAP_POPULATE will not be + // used. Population now disabled by default; opt-in via SVS_ENABLE_MAP_POPULATE. + const bool enable_populate = (std::getenv("SVS_ENABLE_MAP_POPULATE") != nullptr); + // Heuristic: Skip attempting huge pages whose rounded allocation would exceed 2x + // requested. This avoids over-large temporary mappings when the user dataset is + // smaller. for (auto params : hugepage_x86_options) { // Don't fallback to huge pages if `force == true`. if (force && params == hugepage_x86_options.back()) { @@ -122,6 +134,11 @@ struct HugepageAllocation { auto flags = params.mmap_flags; sz = lib::round_up_to_multiple_of(bytes, pagesize); + if (sz > 2 * bytes) { + // Skip this page size; too much waste for transient attempt. + continue; + } + int mmap_flags = MAP_PRIVATE; #if defined(MAP_ANONYMOUS) mmap_flags |= MAP_ANONYMOUS; @@ -132,7 +149,9 @@ struct HugepageAllocation { #endif // Add Linux-specific flags #ifdef __linux__ - mmap_flags |= MAP_POPULATE; + if (enable_populate) { + mmap_flags |= MAP_POPULATE; + } #endif // __linux__ ptr = mmap(nullptr, sz, PROT_READ | PROT_WRITE, mmap_flags | flags, -1, 0); @@ -362,8 +381,9 @@ template class MMapPtr { // Make the conversion from "void" to non-void "explicit" to avoid nasty unexpected // implicit conversions. template - requires(std::is_same_v || std::is_same_v) - MMapPtr(MMapPtr&& other) noexcept + requires(std::is_same_v || std::is_same_v) + MMapPtr(MMapPtr&& other) + noexcept : ptr_{reinterpret_cast(other.ptr_)} , base_{other.base_} , size_{other.size_} { @@ -517,9 +537,13 @@ class MemoryMapper { int mmap_flags = MAP_SHARED; // Accessible from all processes // Add Linux-specific flags #ifdef __linux__ + // Respect environment variable disabling population. + const bool enable_populate = (std::getenv("SVS_ENABLE_MAP_POPULATE") != nullptr); mmap_flags |= MAP_NORESERVE; // Don't reserve space in DRAM for this until used - mmap_flags |= MAP_POPULATE; // Populate page table entries in the DRAM -#endif // __linux__ + if (enable_populate) { + mmap_flags |= MAP_POPULATE; // Populate page table entries in the DRAM + } +#endif // __linux__ void* base = ::mmap( nullptr, bytes.value(), mmap_permissions(permission_), mmap_flags, fd, 0 @@ -536,7 +560,9 @@ class MemoryMapper { namespace detail { template -concept HasValueType = requires { typename Alloc::value_type; }; +concept HasValueType = requires { + typename Alloc::value_type; +}; // clang-format off template @@ -601,10 +627,9 @@ template class AllocatorHandle { using value_type = T; template - explicit AllocatorHandle(Impl&& impl) - requires(!std::is_same_v) && - std::is_rvalue_reference_v && - std::is_same_v + explicit AllocatorHandle(Impl&& impl + ) requires(!std::is_same_v) && + std::is_rvalue_reference_v&& std::is_same_v : impl_{new AllocatorImpl(std::move(impl))} {} AllocatorHandle() {} @@ -622,25 +647,23 @@ template class AllocatorHandle { template friend class AllocatorHandle; template - AllocatorHandle(const AllocatorHandle& other) - requires std::is_same_v && (!std::is_same_v) + AllocatorHandle(const AllocatorHandle& other) requires std::is_same_v && + (!std::is_same_v) : impl_{other.impl_->rebind_float()} {} template - AllocatorHandle(const AllocatorHandle& other) - requires std::is_same_v && (!std::is_same_v) + AllocatorHandle(const AllocatorHandle& other) requires std::is_same_v && + (!std::is_same_v) : impl_{other.impl_->rebind_float16()} {} template - AllocatorHandle& operator=(const AllocatorHandle& other) - requires std::is_same_v && (!std::is_same_v) - { + AllocatorHandle& operator=(const AllocatorHandle& other + ) requires std::is_same_v &&(!std::is_same_v) { impl_.reset(other.impl_->rebind_float()); return *this; } template - AllocatorHandle& operator=(const AllocatorHandle& other) - requires std::is_same_v && (!std::is_same_v) - { + AllocatorHandle& operator=(const AllocatorHandle& other + ) requires std::is_same_v &&(!std::is_same_v) { impl_.reset(other.impl_->rebind_float16()); return *this; }