From 2f690667488c94d6553540292127a5c2fd5bd6cd Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 19 Nov 2024 19:59:34 +0100 Subject: [PATCH 01/23] use std::swap --- runtime/alloc/arena.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index c4384642e..e69b3e4ec 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -161,26 +161,20 @@ arena::arena_resize_last_alloc(ssize_t increase) { } __attribute__((always_inline)) void arena::arena_swap_and_clear() { - char *tmp = first_block; - first_block = first_collection_block; - first_collection_block = tmp; - size_t tmp2 = num_blocks; - num_blocks = num_collection_blocks; - num_collection_blocks = tmp2; + std::swap(first_block, first_collection_block); + std::swap(num_blocks, num_collection_blocks); allocation_semispace_id = ~allocation_semispace_id; arena_clear(); } __attribute__((always_inline)) void arena::arena_clear() { - block = first_block ? first_block + sizeof(arena::memory_block_header) - : nullptr; + block = first_block ? first_block + sizeof(arena::memory_block_header) : nullptr; block_start = first_block; block_end = first_block ? first_block + BLOCK_SIZE : nullptr; } __attribute__((always_inline)) char *arena::arena_start_ptr() const { - return first_block ? first_block + sizeof(arena::memory_block_header) - : nullptr; + return first_block ? first_block + sizeof(arena::memory_block_header) : nullptr; } __attribute__((always_inline)) char **arena::arena_end_ptr() { From fa25f36fa5bea57383e3108032226be98ea9d2ff Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 19 Nov 2024 22:40:20 +0100 Subject: [PATCH 02/23] make megabye_malloc() private member function; added data members current_block_ptr and collection_block_ptr; added initialization for data members --- include/runtime/arena.h | 24 +++++++++++++++--------- runtime/alloc/arena.cpp | 13 ++++++------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index fc72155fb..53710a2d6 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -73,7 +73,7 @@ class arena { // Return value: the address allocated in the arena after size bytes from the // starting pointer, or 0 if this is equal to the 3rd argument. static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr); - + // Returns the ID of the semispace where the given address was allocated. // The behavior is undefined if called with an address that has not been // allocated within an arena. @@ -85,20 +85,25 @@ class arena { char semispace; }; + void *megabyte_malloc(); + void fresh_block(); static memory_block_header *mem_block_header(void *ptr); // helper function for `kore_arena_alloc`. Do not call directly. void *do_alloc_slow(size_t requested); - char *first_block; // beginning of first block - char *block; // where allocations are being made in current block - char *block_start; // start of current block - char *block_end; // 1 past end of current block - char *first_collection_block; // beginning of other semispace - size_t num_blocks; // number of blocks in current semispace - size_t num_collection_blocks; // number of blocks in other semispace - char allocation_semispace_id; // id of current semispace + char *current_block_ptr = nullptr; // pointer to current block within hyperblock + char *collection_block_ptr = nullptr; // pointer to current block within collection hyperblock + + char *first_block = nullptr; // beginning of first block + char *block = nullptr; // where allocations are being made in current block + char *block_start = nullptr; // start of current block + char *block_end = nullptr; // 1 past end of current block + char *first_collection_block = nullptr; // beginning of other semispace + size_t num_blocks = 0; // number of blocks in current semispace + size_t num_collection_blocks = 0; // number of blocks in other semispace + char allocation_semispace_id; // id of current semispace }; // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to @@ -130,6 +135,7 @@ inline void *arena::kore_arena_alloc(size_t requested) { requested, block); return result; } + } #endif // ARENA_H diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index e69b3e4ec..3044d53e7 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -34,18 +34,16 @@ arena::get_arena_semispace_id_of_object(void *ptr) { // size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; -static void *megabyte_malloc() { +void *arena::megabyte_malloc() { // // Return pointer to a BLOCK_SIZE chunk of memory with BLOCK_SIZE alignment. // - static thread_local char *currentblock_ptr - = nullptr; // char* rather than void* to permit pointer arithmetic - if (currentblock_ptr) { + if (current_block_ptr) { // // We expect an page fault due to not being able to map physical memory to this block or the // process to be killed by the OOM killer long before we run off the end of our address space. // - currentblock_ptr += BLOCK_SIZE; + current_block_ptr += BLOCK_SIZE; } else { // // First call - need to reserve the address space. @@ -68,10 +66,10 @@ static void *megabyte_malloc() { // We don't worry about unused address space either side of our aligned address space because there will be no // memory mapped to it. // - currentblock_ptr = reinterpret_cast( + current_block_ptr = reinterpret_cast( std::align(BLOCK_SIZE, HYPERBLOCK_SIZE - BLOCK_SIZE, addr, request)); } - return currentblock_ptr; + return current_block_ptr; } #ifdef __MACH__ @@ -163,6 +161,7 @@ arena::arena_resize_last_alloc(ssize_t increase) { __attribute__((always_inline)) void arena::arena_swap_and_clear() { std::swap(first_block, first_collection_block); std::swap(num_blocks, num_collection_blocks); + std::swap(current_block_ptr, collection_block_ptr); allocation_semispace_id = ~allocation_semispace_id; arena_clear(); } From a94b6a916be47c5fc9516bbf796bcd765bfa5b21 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Wed, 20 Nov 2024 03:52:18 +0100 Subject: [PATCH 03/23] make ptr_diff() a simple subtraction --- include/runtime/arena.h | 2 +- runtime/alloc/arena.cpp | 28 ---------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 53710a2d6..0f2acdd95 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -61,7 +61,7 @@ class arena { // by the blocks of that arena. This difference will include blocks containing // sentinel bytes. Undefined behavior will result if the pointers belong to // different arenas. - static ssize_t ptr_diff(char *ptr1, char *ptr2); + static ssize_t ptr_diff(char *ptr1, char *ptr2) { return ptr1 - ptr2; } // Given a starting pointer to an address allocated in an arena and a size in // bytes, this function returns a pointer to an address allocated in the diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 3044d53e7..3eb886044 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -195,34 +195,6 @@ char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { return next_block + sizeof(arena::memory_block_header); } -ssize_t arena::ptr_diff(char *ptr1, char *ptr2) { - if (MEM_BLOCK_START(ptr1) == MEM_BLOCK_START(ptr2)) { - return ptr1 - ptr2; - } - arena::memory_block_header *hdr = mem_block_header(ptr2); - ssize_t result = 0; - while (hdr != mem_block_header(ptr1) && hdr->next_block) { - if (ptr2) { - result += ((char *)hdr + BLOCK_SIZE) - ptr2; - ptr2 = nullptr; - } else { - result += (BLOCK_SIZE - sizeof(arena::memory_block_header)); - } - hdr = (arena::memory_block_header *)hdr->next_block; - } - if (hdr == mem_block_header(ptr1)) { - result += ptr1 - (char *)(hdr + 1); - return result; - } // reached the end of the arena and didn't find the block - // it's possible that the result should be negative, in which - // case the block will have been prior to the block we started - // at. To handle this, we recurse with reversed arguments and - // negate the result. This means that the code might not - // terminate if the two pointers do not belong to the same - // arena. - return -ptr_diff(ptr2, ptr1); -} - size_t arena::arena_size() const { return (num_blocks > num_collection_blocks ? num_blocks : num_collection_blocks) From 259290d2ccd488e238e1920fecec8395854d6de9 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Thu, 21 Nov 2024 03:36:15 +0100 Subject: [PATCH 04/23] reimplement class arena --- include/runtime/arena.h | 48 +++++----- runtime/alloc/arena.cpp | 197 +++++++++++++--------------------------- 2 files changed, 86 insertions(+), 159 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 0f2acdd95..83f41e240 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -8,6 +8,7 @@ extern "C" { + // An arena can be used to allocate objects that can then be deallocated all at // once. class arena { @@ -23,12 +24,12 @@ class arena { // Returns the address of the first byte that belongs in the given arena. // Returns 0 if nothing has been allocated ever in that arena. - char *arena_start_ptr() const; + char *arena_start_ptr() const { return current_addr_ptr ? current_addr_ptr + sizeof(memory_block_header) : nullptr; } // Returns a pointer to a location holding the address of last allocated // byte in the given arena plus 1. // This address is 0 if nothing has been allocated ever in that arena. - char **arena_end_ptr(); + char **arena_end_ptr() { return &allocation_ptr; } // return the total number of allocatable bytes currently in the arena in its // active semispace. @@ -37,19 +38,19 @@ class arena { // Clears the current allocation space by setting its start back to its first // block. It is used during garbage collection to effectively collect all of the // arena. - void arena_clear(); + void arena_clear() { allocation_ptr = arena_start_ptr(); } // Resizes the last allocation as long as the resize does not require a new // block allocation. // Returns the address of the byte following the last newlly allocated byte when // the resize succeeds, returns 0 otherwise. - void *arena_resize_last_alloc(ssize_t increase); + void *arena_resize_last_alloc(ssize_t increase) { return (allocation_ptr += increase); } // Returns the given arena's current collection semispace ID. // Each arena has 2 semispace IDs one equal to the arena ID and the other equal // to the 1's complement of the arena ID. At any time one of these semispaces // is used for allocation and the other is used for collection. - char get_arena_collection_semispace_id() const; + char get_arena_collection_semispace_id() const { return ~allocation_semispace_id; } // Exchanges the current allocation and collection semispaces and clears the new // current allocation semispace by setting its start back to its first block. @@ -85,6 +86,7 @@ class arena { char semispace; }; + void *slow_alloc(size_t requested); void *megabyte_malloc(); void fresh_block(); @@ -93,16 +95,12 @@ class arena { // helper function for `kore_arena_alloc`. Do not call directly. void *do_alloc_slow(size_t requested); - char *current_block_ptr = nullptr; // pointer to current block within hyperblock - char *collection_block_ptr = nullptr; // pointer to current block within collection hyperblock + char *current_addr_ptr = nullptr; // pointer to start of current address space + char *collection_addr_ptr = nullptr; // pointer to start of collection address space + char *current_tripwire = nullptr; // allocating past this triggers slow allocation + char *collection_tripwire = nullptr; // tripwire for collection semispace + char *allocation_ptr = nullptr; // next available location in current semispace - char *first_block = nullptr; // beginning of first block - char *block = nullptr; // where allocations are being made in current block - char *block_start = nullptr; // start of current block - char *block_end = nullptr; // 1 past end of current block - char *first_collection_block = nullptr; // beginning of other semispace - size_t num_blocks = 0; // number of blocks in current semispace - size_t num_collection_blocks = 0; // number of blocks in other semispace char allocation_semispace_id; // id of current semispace }; @@ -125,17 +123,21 @@ extern thread_local bool time_for_collection; size_t get_gc_threshold(); inline void *arena::kore_arena_alloc(size_t requested) { - if (block + requested > block_end) { - return do_alloc_slow(requested); + if (allocation_ptr + requested >= current_tripwire) { + // + // We got close to or past the last location accessed in this address range so far, + // thus we should consider a garbage collection so we don't just keep accessing + // fresh memory without bound. + // This condition holds trivially if current_tripwire == nullptr since + // all pointers are >= nullptr, and this indicates that no address range has been + // reserved for this semispace so far. + // + return slow_alloc(requested); } - void *result = block; - block += requested; - MEM_LOG( - "Allocation at %p (size %zd), next alloc at %p (if it fits)\n", result, - requested, block); + void *result = allocation_ptr; + allocation_ptr += requested; + MEM_LOG("Allocation at %p (size %zd), next alloc at %p\n", result, requested, block); return result; } - } - #endif // ARENA_H diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 3eb886044..a9a159fb9 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -11,17 +11,14 @@ #include "runtime/header.h" extern size_t const VAR_BLOCK_SIZE = BLOCK_SIZE; +size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; + __attribute__((always_inline)) arena::memory_block_header * arena::mem_block_header(void *ptr) { // NOLINTNEXTLINE(*-reinterpret-cast) return reinterpret_cast( - ((uintptr_t)(ptr)-1) & ~(BLOCK_SIZE - 1)); -} - -__attribute__((always_inline)) char -arena::get_arena_collection_semispace_id() const { - return ~allocation_semispace_id; + ((uintptr_t)(ptr)-1) & ~(HYPERBLOCK_SIZE - 1)); } __attribute__((always_inline)) char @@ -29,48 +26,6 @@ arena::get_arena_semispace_id_of_object(void *ptr) { return mem_block_header(ptr)->semispace; } -// -// We will reserve enough address space for 1 million 1MB blocks. Might want to increase this on a > 1TB server. -// -size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; - -void *arena::megabyte_malloc() { - // - // Return pointer to a BLOCK_SIZE chunk of memory with BLOCK_SIZE alignment. - // - if (current_block_ptr) { - // - // We expect an page fault due to not being able to map physical memory to this block or the - // process to be killed by the OOM killer long before we run off the end of our address space. - // - current_block_ptr += BLOCK_SIZE; - } else { - // - // First call - need to reserve the address space. - // - size_t request = HYPERBLOCK_SIZE; - void *addr = mmap( - nullptr, // let OS choose the address - request, // Linux and MacOS both allow up to 64TB - PROT_READ | PROT_WRITE, // read, write but not execute - MAP_ANONYMOUS | MAP_PRIVATE - | MAP_NORESERVE, // allocate address space only - -1, // no file backing - 0); // no offset - if (addr == MAP_FAILED) { - perror("mmap()"); - abort(); - } - // - // We ask for one block worth of address space less than we allocated so alignment will always succeed. - // We don't worry about unused address space either side of our aligned address space because there will be no - // memory mapped to it. - // - current_block_ptr = reinterpret_cast( - std::align(BLOCK_SIZE, HYPERBLOCK_SIZE - BLOCK_SIZE, addr, request)); - } - return current_block_ptr; -} #ifdef __MACH__ // @@ -81,49 +36,6 @@ bool time_for_collection; thread_local bool time_for_collection; #endif -void arena::fresh_block() { - char *next_block = nullptr; - if (block_start == nullptr) { - next_block = (char *)megabyte_malloc(); - first_block = next_block; - auto *next_header = (arena::memory_block_header *)next_block; - next_header->next_block = nullptr; - next_header->semispace = allocation_semispace_id; - num_blocks++; - } else { - next_block = *(char **)block_start; - if (block != block_end) { - if (block_end - block == 8) { - *(uint64_t *)block = NOT_YOUNG_OBJECT_BIT; // 8 bit sentinel value - } else { - *(uint64_t *)block - = block_end - block - 8; // 16-bit or more sentinel value - } - } - if (!next_block) { - MEM_LOG( - "Allocating new block for the first time in arena %d\n", - allocation_semispace_id); - next_block = (char *)megabyte_malloc(); - *(char **)block_start = next_block; - auto *next_header = (arena::memory_block_header *)next_block; - next_header->next_block = nullptr; - next_header->semispace = allocation_semispace_id; - num_blocks++; - time_for_collection = true; - } - } - if (!*(char **)next_block && num_blocks >= get_gc_threshold()) { - time_for_collection = true; - } - block = next_block + sizeof(arena::memory_block_header); - block_start = next_block; - block_end = next_block + BLOCK_SIZE; - MEM_LOG( - "New block at %p (remaining %zd)\n", block, - BLOCK_SIZE - sizeof(arena::memory_block_header)); -} - #ifdef __MACH__ // // thread_local disabled for Apple @@ -133,53 +45,13 @@ bool gc_enabled = true; thread_local bool gc_enabled = true; #endif -__attribute__((noinline)) void *arena::do_alloc_slow(size_t requested) { - MEM_LOG( - "Block at %p too small, %zd remaining but %zd needed\n", block, - block_end - block, requested); - if (requested > BLOCK_SIZE - sizeof(arena::memory_block_header)) { - return malloc(requested); - } - fresh_block(); - void *result = block; - block += requested; - MEM_LOG( - "Allocation at %p (size %zd), next alloc at %p (if it fits)\n", result, - requested, block); - return result; -} - -__attribute__((always_inline)) void * -arena::arena_resize_last_alloc(ssize_t increase) { - if (block + increase <= block_end) { - block += increase; - return block; - } - return nullptr; -} - __attribute__((always_inline)) void arena::arena_swap_and_clear() { - std::swap(first_block, first_collection_block); - std::swap(num_blocks, num_collection_blocks); - std::swap(current_block_ptr, collection_block_ptr); + std::swap(current_addr_ptr, collection_addr_ptr); + std::swap(current_tripwire, collection_tripwire); allocation_semispace_id = ~allocation_semispace_id; arena_clear(); } -__attribute__((always_inline)) void arena::arena_clear() { - block = first_block ? first_block + sizeof(arena::memory_block_header) : nullptr; - block_start = first_block; - block_end = first_block ? first_block + BLOCK_SIZE : nullptr; -} - -__attribute__((always_inline)) char *arena::arena_start_ptr() const { - return first_block ? first_block + sizeof(arena::memory_block_header) : nullptr; -} - -__attribute__((always_inline)) char **arena::arena_end_ptr() { - return █ -} - char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { char *next_ptr = ptr + size; if (next_ptr == arena_end_ptr) { @@ -196,7 +68,60 @@ char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { } size_t arena::arena_size() const { - return (num_blocks > num_collection_blocks ? num_blocks - : num_collection_blocks) - * (BLOCK_SIZE - sizeof(arena::memory_block_header)); + size_t current_size = current_addr_ptr ? (BLOCK_SIZE + current_tripwire - current_addr_ptr) : 0; + size_t collection_size = collection_addr_ptr ? (BLOCK_SIZE + collection_tripwire - collection_addr_ptr) : 0; + return std::max(current_size, collection_size); +} + +void *arena::slow_alloc(size_t requested) { + // + // This allocation will push the allocation_ptr beyond the tripwire + // into or past the cushion area between allocation_ptr and the furthest + // allocated location. + // + if (current_tripwire == nullptr) { + // + // No address space has been reserved for this semispace. + // + size_t request = 2 * HYPERBLOCK_SIZE; + void *addr = mmap( + nullptr, // let OS choose the address + request, // Linux and MacOS both allow up to 64TB + PROT_READ | PROT_WRITE, // read, write but not execute + MAP_ANONYMOUS | MAP_PRIVATE + | MAP_NORESERVE, // allocate address space only + -1, // no file backing + 0); // no offset + if (addr == MAP_FAILED) { + perror("mmap()"); + abort(); + } + // + // We allocated 2 * HYPERBLOCK_SIZE worth of address space but we're only going to use 1, aligned on a + // HYPERBLOCK_SIZE boundry. This is so we can get the start of the hyperblock by masking any address within it. + // We don't worry about unused address space either side of our aligned address space because there will be no + // memory mapped to it. + // + current_addr_ptr = reinterpret_cast( + std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); + auto *header = (arena::memory_block_header *) current_addr_ptr; + header->next_block = nullptr; + header->semispace = allocation_semispace_id; + allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header); + current_tripwire = current_addr_ptr + BLOCK_SIZE; + } + else { + // + // Need a garbage collection. We also move the tripwire so we don't hit it repeatedly. + // We always move the tripwire to a BLOCK_SIZE boundry. + // + time_for_collection = true; + while (allocation_ptr + requested >= current_tripwire) + current_tripwire += BLOCK_SIZE; + } + + void *result = allocation_ptr; + allocation_ptr += requested; + MEM_LOG("Slow allocation at %p (size %zd), next alloc at %p\n", result, requested, block); + return result; } From 41a9975e198e3e05dee0159fff08aaeb1543d221 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Fri, 22 Nov 2024 00:12:26 +0100 Subject: [PATCH 05/23] deleted decl for megabyte_malloc(), fresh_block(), code cleaning --- include/runtime/arena.h | 5 ----- runtime/alloc/arena.cpp | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 83f41e240..1e1fe894c 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -87,14 +87,9 @@ class arena { }; void *slow_alloc(size_t requested); - void *megabyte_malloc(); - void fresh_block(); static memory_block_header *mem_block_header(void *ptr); - // helper function for `kore_arena_alloc`. Do not call directly. - void *do_alloc_slow(size_t requested); - char *current_addr_ptr = nullptr; // pointer to start of current address space char *collection_addr_ptr = nullptr; // pointer to start of collection address space char *current_tripwire = nullptr; // allocating past this triggers slow allocation diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index a9a159fb9..f605dac3f 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -104,7 +104,7 @@ void *arena::slow_alloc(size_t requested) { // current_addr_ptr = reinterpret_cast( std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); - auto *header = (arena::memory_block_header *) current_addr_ptr; + memory_block_header *header = reinterpret_cast(current_addr_ptr); header->next_block = nullptr; header->semispace = allocation_semispace_id; allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header); From 30a0b71c73582c70998915e5a081dd6239a3d89e Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Fri, 22 Nov 2024 02:07:36 +0100 Subject: [PATCH 06/23] added initialize_semispace() to avoid relying on UB for initialization; moved some inline functions from arena.cpp to arena.h; make struct memory_block_header in a union; fix comments --- include/runtime/arena.h | 41 ++++++++++++---- runtime/alloc/arena.cpp | 101 +++++++++++++++++----------------------- 2 files changed, 75 insertions(+), 67 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 1e1fe894c..6c888092b 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -2,24 +2,24 @@ #define ARENA_H #include +#include +#include #include #include "runtime/alloc.h" extern "C" { +size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; // An arena can be used to allocate objects that can then be deallocated all at // once. class arena { public: - arena(char id) - : allocation_semispace_id(id) { } + arena(char id) : allocation_semispace_id(id) { initialize_semispace(); } // Allocates the requested number of bytes as a contiguous region and returns a // pointer to the first allocated byte. - // If called with requested size greater than the maximun single allocation - // size, the space is allocated in a general (not garbage collected pool). void *kore_arena_alloc(size_t requested); // Returns the address of the first byte that belongs in the given arena. @@ -42,8 +42,7 @@ class arena { // Resizes the last allocation as long as the resize does not require a new // block allocation. - // Returns the address of the byte following the last newlly allocated byte when - // the resize succeeds, returns 0 otherwise. + // Returns the address of the byte following the last newlly allocated byte. void *arena_resize_last_alloc(ssize_t increase) { return (allocation_ptr += increase); } // Returns the given arena's current collection semispace ID. @@ -81,14 +80,22 @@ class arena { static char get_arena_semispace_id_of_object(void *ptr); private: - struct memory_block_header { - char *next_block; + union memory_block_header { + // + // Currently the header just holds the semispace id. But we need it to be a + // multiple of sizeof(char*) for alignment purposes so we add a dummy char*. + // char semispace; + char *alignment_dummy; }; void *slow_alloc(size_t requested); + void initialize_semispace(); - static memory_block_header *mem_block_header(void *ptr); + static memory_block_header *mem_block_header(void *ptr) { + uintptr_t address = reinterpret_cast(ptr); + return reinterpret_cast((address - 1) & ~(HYPERBLOCK_SIZE - 1)); + } char *current_addr_ptr = nullptr; // pointer to start of current address space char *collection_addr_ptr = nullptr; // pointer to start of collection address space @@ -134,5 +141,21 @@ inline void *arena::kore_arena_alloc(size_t requested) { MEM_LOG("Allocation at %p (size %zd), next alloc at %p\n", result, requested, block); return result; } + +inline void arena::arena_swap_and_clear() { + std::swap(current_addr_ptr, collection_addr_ptr); + std::swap(current_tripwire, collection_tripwire); + allocation_semispace_id = ~allocation_semispace_id; + if (current_addr_ptr == nullptr) + { + // + // The other semispace hasn't be initialized yet. + // + void initialize_semispace(); + } + else + arena_clear(); +} + } #endif // ARENA_H diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index f605dac3f..41c1ebbc6 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -11,15 +11,6 @@ #include "runtime/header.h" extern size_t const VAR_BLOCK_SIZE = BLOCK_SIZE; -size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; - - -__attribute__((always_inline)) arena::memory_block_header * -arena::mem_block_header(void *ptr) { - // NOLINTNEXTLINE(*-reinterpret-cast) - return reinterpret_cast( - ((uintptr_t)(ptr)-1) & ~(HYPERBLOCK_SIZE - 1)); -} __attribute__((always_inline)) char arena::get_arena_semispace_id_of_object(void *ptr) { @@ -45,12 +36,6 @@ bool gc_enabled = true; thread_local bool gc_enabled = true; #endif -__attribute__((always_inline)) void arena::arena_swap_and_clear() { - std::swap(current_addr_ptr, collection_addr_ptr); - std::swap(current_tripwire, collection_tripwire); - allocation_semispace_id = ~allocation_semispace_id; - arena_clear(); -} char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { char *next_ptr = ptr + size; @@ -73,53 +58,53 @@ size_t arena::arena_size() const { return std::max(current_size, collection_size); } -void *arena::slow_alloc(size_t requested) { +void arena::initialize_semispace() { // - // This allocation will push the allocation_ptr beyond the tripwire - // into or past the cushion area between allocation_ptr and the furthest - // allocated location. + // Current semispace is uninitialized so mmap() a big chuck of address space. // - if (current_tripwire == nullptr) { - // - // No address space has been reserved for this semispace. - // - size_t request = 2 * HYPERBLOCK_SIZE; - void *addr = mmap( - nullptr, // let OS choose the address - request, // Linux and MacOS both allow up to 64TB - PROT_READ | PROT_WRITE, // read, write but not execute - MAP_ANONYMOUS | MAP_PRIVATE - | MAP_NORESERVE, // allocate address space only - -1, // no file backing - 0); // no offset - if (addr == MAP_FAILED) { - perror("mmap()"); - abort(); - } - // - // We allocated 2 * HYPERBLOCK_SIZE worth of address space but we're only going to use 1, aligned on a - // HYPERBLOCK_SIZE boundry. This is so we can get the start of the hyperblock by masking any address within it. - // We don't worry about unused address space either side of our aligned address space because there will be no - // memory mapped to it. - // - current_addr_ptr = reinterpret_cast( - std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); - memory_block_header *header = reinterpret_cast(current_addr_ptr); - header->next_block = nullptr; - header->semispace = allocation_semispace_id; - allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header); - current_tripwire = current_addr_ptr + BLOCK_SIZE; - } - else { - // - // Need a garbage collection. We also move the tripwire so we don't hit it repeatedly. - // We always move the tripwire to a BLOCK_SIZE boundry. - // - time_for_collection = true; - while (allocation_ptr + requested >= current_tripwire) - current_tripwire += BLOCK_SIZE; + size_t request = 2 * HYPERBLOCK_SIZE; + void *addr = mmap( + nullptr, // let OS choose the address + request, // Linux and MacOS both allow up to 64TB + PROT_READ | PROT_WRITE, // read, write but not execute + MAP_ANONYMOUS | MAP_PRIVATE + | MAP_NORESERVE, // allocate address space only + -1, // no file backing + 0); // no offset + if (addr == MAP_FAILED) { + perror("mmap()"); + abort(); } + // + // We allocated 2 * HYPERBLOCK_SIZE worth of address space but we're only going to use 1, aligned on a + // HYPERBLOCK_SIZE boundry. This is so we can get the start of the hyperblock by masking any address within it. + // We don't worry about unused address space either side of our aligned address space because there will be no + // memory mapped to it. + // + current_addr_ptr = reinterpret_cast(std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); + // + // We put a memory_block_header at the beginning so we can identify the semispace a pointer belongs to + // id by masking off the low bits to access this memory_block_header. + // + memory_block_header *header = reinterpret_cast(current_addr_ptr); + header->semispace = allocation_semispace_id; + allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header); + // + // We set the tripwire for this space so we get a slow_alloc() when we pass BLOCK_SIZE of memory + // allocated from this space. + // + current_tripwire = current_addr_ptr + BLOCK_SIZE; +} +void *arena::slow_alloc(size_t requested) { + // + // Need a garbage collection. We also move the tripwire so we don't hit it repeatedly. + // We always move the tripwire to a BLOCK_SIZE boundry. + // + time_for_collection = true; + while (allocation_ptr + requested >= current_tripwire) + current_tripwire += BLOCK_SIZE; + void *result = allocation_ptr; allocation_ptr += requested; MEM_LOG("Slow allocation at %p (size %zd), next alloc at %p\n", result, requested, block); From 62a8ee77a71d28ce6dae2ba16702958e7223d57e Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Fri, 22 Nov 2024 03:30:43 +0100 Subject: [PATCH 07/23] keep track of notional number of blocks in each semispace --- include/runtime/arena.h | 39 ++++++++++++++++++++++++++------------- runtime/alloc/arena.cpp | 31 ++++++++++--------------------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 6c888092b..39f8516ae 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "runtime/alloc.h" @@ -33,12 +34,12 @@ class arena { // return the total number of allocatable bytes currently in the arena in its // active semispace. - size_t arena_size() const; + size_t arena_size() const { return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks); } // Clears the current allocation space by setting its start back to its first // block. It is used during garbage collection to effectively collect all of the // arena. - void arena_clear() { allocation_ptr = arena_start_ptr(); } + void arena_clear(); // Resizes the last allocation as long as the resize does not require a new // block allocation. @@ -97,13 +98,19 @@ class arena { return reinterpret_cast((address - 1) & ~(HYPERBLOCK_SIZE - 1)); } - char *current_addr_ptr = nullptr; // pointer to start of current address space - char *collection_addr_ptr = nullptr; // pointer to start of collection address space - char *current_tripwire = nullptr; // allocating past this triggers slow allocation - char *collection_tripwire = nullptr; // tripwire for collection semispace - char *allocation_ptr = nullptr; // next available location in current semispace - + // + // Current semispace where allocations are being made. + // + char *current_addr_ptr; // pointer to start of current address space + char *allocation_ptr; // next available location in current semispace + char *tripwire; // allocating past this triggers slow allocation + size_t num_blocks; // notional number of BLOCK_SIZE blocks in current semispace char allocation_semispace_id; // id of current semispace + // + // Semispace where allocations will be made during and after garbage collect. + // + char *collection_addr_ptr = nullptr; // pointer to start of collection address space + size_t num_collection_blocks = 0; // notional number of BLOCK_SIZE blocks in collection semispace }; // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to @@ -125,14 +132,11 @@ extern thread_local bool time_for_collection; size_t get_gc_threshold(); inline void *arena::kore_arena_alloc(size_t requested) { - if (allocation_ptr + requested >= current_tripwire) { + if (allocation_ptr + requested >= tripwire) { // // We got close to or past the last location accessed in this address range so far, // thus we should consider a garbage collection so we don't just keep accessing // fresh memory without bound. - // This condition holds trivially if current_tripwire == nullptr since - // all pointers are >= nullptr, and this indicates that no address range has been - // reserved for this semispace so far. // return slow_alloc(requested); } @@ -142,9 +146,18 @@ inline void *arena::kore_arena_alloc(size_t requested) { return result; } +inline void arena::arena_clear() { + // + // We set the allocation pointer to the first available address and set + // the tripwire to one block before the notional end. + // + allocation_ptr = arena_start_ptr(); + tripwire = current_addr_ptr + (num_blocks - 1) * BLOCK_SIZE; +} + inline void arena::arena_swap_and_clear() { std::swap(current_addr_ptr, collection_addr_ptr); - std::swap(current_tripwire, collection_tripwire); + std::swap(num_blocks, num_collection_blocks); allocation_semispace_id = ~allocation_semispace_id; if (current_addr_ptr == nullptr) { diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 41c1ebbc6..063fe7c26 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -1,4 +1,3 @@ - #include #include #include @@ -17,26 +16,17 @@ arena::get_arena_semispace_id_of_object(void *ptr) { return mem_block_header(ptr)->semispace; } - -#ifdef __MACH__ -// -// thread_local disabled for Apple -// -bool time_for_collection; -#else -thread_local bool time_for_collection; -#endif - #ifdef __MACH__ // // thread_local disabled for Apple // +bool time_for_collection = false; bool gc_enabled = true; #else +thread_local bool time_for_collection = false; thread_local bool gc_enabled = true; #endif - char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { char *next_ptr = ptr + size; if (next_ptr == arena_end_ptr) { @@ -52,12 +42,6 @@ char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { return next_block + sizeof(arena::memory_block_header); } -size_t arena::arena_size() const { - size_t current_size = current_addr_ptr ? (BLOCK_SIZE + current_tripwire - current_addr_ptr) : 0; - size_t collection_size = collection_addr_ptr ? (BLOCK_SIZE + collection_tripwire - collection_addr_ptr) : 0; - return std::max(current_size, collection_size); -} - void arena::initialize_semispace() { // // Current semispace is uninitialized so mmap() a big chuck of address space. @@ -93,7 +77,8 @@ void arena::initialize_semispace() { // We set the tripwire for this space so we get a slow_alloc() when we pass BLOCK_SIZE of memory // allocated from this space. // - current_tripwire = current_addr_ptr + BLOCK_SIZE; + tripwire = current_addr_ptr + BLOCK_SIZE; + num_blocks = 2; } void *arena::slow_alloc(size_t requested) { @@ -102,11 +87,15 @@ void *arena::slow_alloc(size_t requested) { // We always move the tripwire to a BLOCK_SIZE boundry. // time_for_collection = true; - while (allocation_ptr + requested >= current_tripwire) - current_tripwire += BLOCK_SIZE; + tripwire = current_addr_ptr + num_blocks * BLOCK_SIZE; // won't trigger again until after gc void *result = allocation_ptr; allocation_ptr += requested; + // + // Set number of notional blocks that will have be written to after memory is used. + // + num_blocks = (allocation_ptr - current_addr_ptr - 1) / BLOCK_SIZE + 1; + MEM_LOG("Slow allocation at %p (size %zd), next alloc at %p\n", result, requested, block); return result; } From 3d5ebe81aa7b80010c3abf97503f0a90390016ce Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Sat, 23 Nov 2024 03:44:22 +0100 Subject: [PATCH 08/23] more accurate simulation of old collection policy; deleted slow_alloc(); fix for move_ptr() --- include/runtime/arena.h | 38 +++++++++++++++++++++++++++++--------- runtime/alloc/arena.cpp | 34 ++++++++-------------------------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 39f8516ae..fa260dd94 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -32,9 +32,10 @@ class arena { // This address is 0 if nothing has been allocated ever in that arena. char **arena_end_ptr() { return &allocation_ptr; } + // return the total number of allocatable bytes currently in the arena in its // active semispace. - size_t arena_size() const { return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks); } + size_t arena_size() const { update_num_blocks(); return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks); } // Clears the current allocation space by setting its start back to its first // block. It is used during garbage collection to effectively collect all of the @@ -90,7 +91,19 @@ class arena { char *alignment_dummy; }; - void *slow_alloc(size_t requested); + // + // We update the number of 1MB blocks actually written to, only when we need this value, + // or before a garbage collection rather than trying to determine when we write to a fresh block. + // + void update_num_blocks() const { + // + // Calculate how many 1M blocks of the current arena we used. + // + size_t num_used_blocks = (allocation_ptr - current_addr_ptr - 1) / BLOCK_SIZE + 1; + if (num_used_blocks > num_blocks) + num_blocks = num_used_blocks; + } + void initialize_semispace(); static memory_block_header *mem_block_header(void *ptr) { @@ -104,7 +117,7 @@ class arena { char *current_addr_ptr; // pointer to start of current address space char *allocation_ptr; // next available location in current semispace char *tripwire; // allocating past this triggers slow allocation - size_t num_blocks; // notional number of BLOCK_SIZE blocks in current semispace + mutable size_t num_blocks; // notional number of BLOCK_SIZE blocks in current semispace char allocation_semispace_id; // id of current semispace // // Semispace where allocations will be made during and after garbage collect. @@ -135,10 +148,11 @@ inline void *arena::kore_arena_alloc(size_t requested) { if (allocation_ptr + requested >= tripwire) { // // We got close to or past the last location accessed in this address range so far, - // thus we should consider a garbage collection so we don't just keep accessing - // fresh memory without bound. + // depending on the requested size and tripwire setting. This triggers a garbage + // collect when allowed. // - return slow_alloc(requested); + time_for_collection = true; + tripwire = current_addr_ptr + HYPERBLOCK_SIZE; // won't trigger again until arena swap } void *result = allocation_ptr; allocation_ptr += requested; @@ -148,14 +162,20 @@ inline void *arena::kore_arena_alloc(size_t requested) { inline void arena::arena_clear() { // - // We set the allocation pointer to the first available address and set - // the tripwire to one block before the notional end. + // We set the allocation pointer to the first available address. // allocation_ptr = arena_start_ptr(); - tripwire = current_addr_ptr + (num_blocks - 1) * BLOCK_SIZE; + // + // If the number of blocks we've touched is >= threshold, we want to trigger + // a garbage collection if we get within 1 block of the end of this area. + // Otherwise we only want to generate a garbage collect if we allocate off the + // end of this area. + // + tripwire = current_addr_ptr + (num_blocks - (num_blocks >= get_gc_threshold())) * BLOCK_SIZE; } inline void arena::arena_swap_and_clear() { + update_num_blocks(); // so we save the correct number of touched blocks std::swap(current_addr_ptr, collection_addr_ptr); std::swap(num_blocks, num_collection_blocks); allocation_semispace_id = ~allocation_semispace_id; diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 063fe7c26..0cfbb5ca0 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -35,11 +35,12 @@ char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { if (next_ptr != MEM_BLOCK_START(ptr) + BLOCK_SIZE) { return next_ptr; } - char *next_block = *(char **)MEM_BLOCK_START(ptr); - if (!next_block) { - return nullptr; - } - return next_block + sizeof(arena::memory_block_header); + //char *next_block = *(char **)MEM_BLOCK_START(ptr); + //if (!next_block) { + // return nullptr; + //} + //return next_block + sizeof(arena::memory_block_header); + return MEM_BLOCK_START(ptr) + BLOCK_SIZE; } void arena::initialize_semispace() { @@ -74,28 +75,9 @@ void arena::initialize_semispace() { header->semispace = allocation_semispace_id; allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header); // - // We set the tripwire for this space so we get a slow_alloc() when we pass BLOCK_SIZE of memory + // We set the tripwire for this space so we get trigger a garbage collection when we pass BLOCK_SIZE of memory // allocated from this space. // tripwire = current_addr_ptr + BLOCK_SIZE; - num_blocks = 2; -} - -void *arena::slow_alloc(size_t requested) { - // - // Need a garbage collection. We also move the tripwire so we don't hit it repeatedly. - // We always move the tripwire to a BLOCK_SIZE boundry. - // - time_for_collection = true; - tripwire = current_addr_ptr + num_blocks * BLOCK_SIZE; // won't trigger again until after gc - - void *result = allocation_ptr; - allocation_ptr += requested; - // - // Set number of notional blocks that will have be written to after memory is used. - // - num_blocks = (allocation_ptr - current_addr_ptr - 1) / BLOCK_SIZE + 1; - - MEM_LOG("Slow allocation at %p (size %zd), next alloc at %p\n", result, requested, block); - return result; + num_blocks = 1; } From 31321ea885e384a591e0612c82ab8a6381bbe5b4 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 26 Nov 2024 00:47:53 +0100 Subject: [PATCH 09/23] used just a pointer addition for move_ptr() --- runtime/alloc/arena.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 0cfbb5ca0..9a02a8274 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -28,6 +28,8 @@ thread_local bool gc_enabled = true; #endif char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { + return ptr + size; + /* char *next_ptr = ptr + size; if (next_ptr == arena_end_ptr) { return nullptr; @@ -35,12 +37,12 @@ char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { if (next_ptr != MEM_BLOCK_START(ptr) + BLOCK_SIZE) { return next_ptr; } - //char *next_block = *(char **)MEM_BLOCK_START(ptr); - //if (!next_block) { - // return nullptr; - //} - //return next_block + sizeof(arena::memory_block_header); - return MEM_BLOCK_START(ptr) + BLOCK_SIZE; + char *next_block = *(char **)MEM_BLOCK_START(ptr); + if (!next_block) { + return nullptr; + } + return next_block + sizeof(arena::memory_block_header); + */ } void arena::initialize_semispace() { From f3972a9f7077a5f5668f43d8e2b01c2aa10f78d0 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 26 Nov 2024 04:39:49 +0100 Subject: [PATCH 10/23] made move_ptr() inline and added check for end of allocation --- include/runtime/arena.h | 7 +++++-- runtime/alloc/arena.cpp | 18 ------------------ 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index fa260dd94..43035019d 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -74,8 +74,11 @@ class arena { // 3rd argument: the address of last allocated byte in the arena plus 1 // Return value: the address allocated in the arena after size bytes from the // starting pointer, or 0 if this is equal to the 3rd argument. - static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr); - + static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { + char *next_ptr = ptr + size; + return (next_ptr == arena_end_ptr) ? 0 : next_ptr; + } + // Returns the ID of the semispace where the given address was allocated. // The behavior is undefined if called with an address that has not been // allocated within an arena. diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 9a02a8274..41fbbb129 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -27,24 +27,6 @@ thread_local bool time_for_collection = false; thread_local bool gc_enabled = true; #endif -char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { - return ptr + size; - /* - char *next_ptr = ptr + size; - if (next_ptr == arena_end_ptr) { - return nullptr; - } - if (next_ptr != MEM_BLOCK_START(ptr) + BLOCK_SIZE) { - return next_ptr; - } - char *next_block = *(char **)MEM_BLOCK_START(ptr); - if (!next_block) { - return nullptr; - } - return next_block + sizeof(arena::memory_block_header); - */ -} - void arena::initialize_semispace() { // // Current semispace is uninitialized so mmap() a big chuck of address space. From d8ef882dbf9728703e1978b5be67f3dbafe82401 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 26 Nov 2024 20:42:21 +0100 Subject: [PATCH 11/23] disable garbage collection to try and isolate bug --- include/runtime/arena.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 43035019d..374345be5 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -154,7 +154,7 @@ inline void *arena::kore_arena_alloc(size_t requested) { // depending on the requested size and tripwire setting. This triggers a garbage // collect when allowed. // - time_for_collection = true; + //time_for_collection = true; tripwire = current_addr_ptr + HYPERBLOCK_SIZE; // won't trigger again until arena swap } void *result = allocation_ptr; From 4318df9f279d30b9dc44cba74b5e920e1bbc4abb Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 26 Nov 2024 23:22:45 +0100 Subject: [PATCH 12/23] re-enabled garbage collection - inifinte loop problem has disappeared --- include/runtime/arena.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 374345be5..43035019d 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -154,7 +154,7 @@ inline void *arena::kore_arena_alloc(size_t requested) { // depending on the requested size and tripwire setting. This triggers a garbage // collect when allowed. // - //time_for_collection = true; + time_for_collection = true; tripwire = current_addr_ptr + HYPERBLOCK_SIZE; // won't trigger again until arena swap } void *result = allocation_ptr; From 7a2df991cf4b82f4eb34fe88902c4e3c816aa074 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Wed, 27 Nov 2024 04:15:42 +0100 Subject: [PATCH 13/23] fixed formatting --- include/runtime/arena.h | 80 +++++++++++++++++++++++++---------------- runtime/alloc/arena.cpp | 20 ++++++----- 2 files changed, 60 insertions(+), 40 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 43035019d..464cc60f0 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -1,11 +1,11 @@ #ifndef ARENA_H #define ARENA_H +#include #include #include -#include -#include #include +#include #include "runtime/alloc.h" @@ -17,7 +17,10 @@ size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; // once. class arena { public: - arena(char id) : allocation_semispace_id(id) { initialize_semispace(); } + arena(char id) + : allocation_semispace_id(id) { + initialize_semispace(); + } // Allocates the requested number of bytes as a contiguous region and returns a // pointer to the first allocated byte. @@ -25,17 +28,22 @@ class arena { // Returns the address of the first byte that belongs in the given arena. // Returns 0 if nothing has been allocated ever in that arena. - char *arena_start_ptr() const { return current_addr_ptr ? current_addr_ptr + sizeof(memory_block_header) : nullptr; } + char *arena_start_ptr() const { + return current_addr_ptr ? current_addr_ptr + sizeof(memory_block_header) + : nullptr; + } // Returns a pointer to a location holding the address of last allocated // byte in the given arena plus 1. // This address is 0 if nothing has been allocated ever in that arena. char **arena_end_ptr() { return &allocation_ptr; } - // return the total number of allocatable bytes currently in the arena in its // active semispace. - size_t arena_size() const { update_num_blocks(); return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks); } + size_t arena_size() const { + update_num_blocks(); + return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks); + } // Clears the current allocation space by setting its start back to its first // block. It is used during garbage collection to effectively collect all of the @@ -45,13 +53,17 @@ class arena { // Resizes the last allocation as long as the resize does not require a new // block allocation. // Returns the address of the byte following the last newlly allocated byte. - void *arena_resize_last_alloc(ssize_t increase) { return (allocation_ptr += increase); } + void *arena_resize_last_alloc(ssize_t increase) { + return (allocation_ptr += increase); + } // Returns the given arena's current collection semispace ID. // Each arena has 2 semispace IDs one equal to the arena ID and the other equal // to the 1's complement of the arena ID. At any time one of these semispaces // is used for allocation and the other is used for collection. - char get_arena_collection_semispace_id() const { return ~allocation_semispace_id; } + char get_arena_collection_semispace_id() const { + return ~allocation_semispace_id; + } // Exchanges the current allocation and collection semispaces and clears the new // current allocation semispace by setting its start back to its first block. @@ -102,31 +114,36 @@ class arena { // // Calculate how many 1M blocks of the current arena we used. // - size_t num_used_blocks = (allocation_ptr - current_addr_ptr - 1) / BLOCK_SIZE + 1; + size_t num_used_blocks + = (allocation_ptr - current_addr_ptr - 1) / BLOCK_SIZE + 1; if (num_used_blocks > num_blocks) num_blocks = num_used_blocks; } void initialize_semispace(); - + static memory_block_header *mem_block_header(void *ptr) { uintptr_t address = reinterpret_cast(ptr); - return reinterpret_cast((address - 1) & ~(HYPERBLOCK_SIZE - 1)); + return reinterpret_cast( + (address - 1) & ~(HYPERBLOCK_SIZE - 1)); } // // Current semispace where allocations are being made. // - char *current_addr_ptr; // pointer to start of current address space - char *allocation_ptr; // next available location in current semispace - char *tripwire; // allocating past this triggers slow allocation - mutable size_t num_blocks; // notional number of BLOCK_SIZE blocks in current semispace - char allocation_semispace_id; // id of current semispace + char *current_addr_ptr; // pointer to start of current address space + char *allocation_ptr; // next available location in current semispace + char *tripwire; // allocating past this triggers slow allocation + mutable size_t + num_blocks; // notional number of BLOCK_SIZE blocks in current semispace + char allocation_semispace_id; // id of current semispace // // Semispace where allocations will be made during and after garbage collect. // - char *collection_addr_ptr = nullptr; // pointer to start of collection address space - size_t num_collection_blocks = 0; // notional number of BLOCK_SIZE blocks in collection semispace + char *collection_addr_ptr + = nullptr; // pointer to start of collection address space + size_t num_collection_blocks + = 0; // notional number of BLOCK_SIZE blocks in collection semispace }; // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to @@ -155,11 +172,14 @@ inline void *arena::kore_arena_alloc(size_t requested) { // collect when allowed. // time_for_collection = true; - tripwire = current_addr_ptr + HYPERBLOCK_SIZE; // won't trigger again until arena swap + tripwire = current_addr_ptr + + HYPERBLOCK_SIZE; // won't trigger again until arena swap } void *result = allocation_ptr; allocation_ptr += requested; - MEM_LOG("Allocation at %p (size %zd), next alloc at %p\n", result, requested, block); + MEM_LOG( + "Allocation at %p (size %zd), next alloc at %p\n", result, requested, + block); return result; } @@ -174,24 +194,22 @@ inline void arena::arena_clear() { // Otherwise we only want to generate a garbage collect if we allocate off the // end of this area. // - tripwire = current_addr_ptr + (num_blocks - (num_blocks >= get_gc_threshold())) * BLOCK_SIZE; + tripwire = current_addr_ptr + + (num_blocks - (num_blocks >= get_gc_threshold())) * BLOCK_SIZE; } inline void arena::arena_swap_and_clear() { - update_num_blocks(); // so we save the correct number of touched blocks + update_num_blocks(); // so we save the correct number of touched blocks std::swap(current_addr_ptr, collection_addr_ptr); std::swap(num_blocks, num_collection_blocks); allocation_semispace_id = ~allocation_semispace_id; - if (current_addr_ptr == nullptr) - { - // - // The other semispace hasn't be initialized yet. - // - void initialize_semispace(); - } - else + if (current_addr_ptr == nullptr) { + // + // The other semispace hasn't be initialized yet. + // + void initialize_semispace(); + } else arena_clear(); } - } #endif // ARENA_H diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 41fbbb129..fe0bfce45 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -33,13 +33,13 @@ void arena::initialize_semispace() { // size_t request = 2 * HYPERBLOCK_SIZE; void *addr = mmap( - nullptr, // let OS choose the address - request, // Linux and MacOS both allow up to 64TB - PROT_READ | PROT_WRITE, // read, write but not execute - MAP_ANONYMOUS | MAP_PRIVATE - | MAP_NORESERVE, // allocate address space only - -1, // no file backing - 0); // no offset + nullptr, // let OS choose the address + request, // Linux and MacOS both allow up to 64TB + PROT_READ | PROT_WRITE, // read, write but not execute + MAP_ANONYMOUS | MAP_PRIVATE + | MAP_NORESERVE, // allocate address space only + -1, // no file backing + 0); // no offset if (addr == MAP_FAILED) { perror("mmap()"); abort(); @@ -50,12 +50,14 @@ void arena::initialize_semispace() { // We don't worry about unused address space either side of our aligned address space because there will be no // memory mapped to it. // - current_addr_ptr = reinterpret_cast(std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); + current_addr_ptr = reinterpret_cast( + std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); // // We put a memory_block_header at the beginning so we can identify the semispace a pointer belongs to // id by masking off the low bits to access this memory_block_header. // - memory_block_header *header = reinterpret_cast(current_addr_ptr); + memory_block_header *header + = reinterpret_cast(current_addr_ptr); header->semispace = allocation_semispace_id; allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header); // From a75b2853db82a86ddd82bab52d0db5bb17fdb7c7 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Thu, 28 Nov 2024 03:14:11 +0100 Subject: [PATCH 14/23] deleted MEM_BLOCK_START; removed corner case from collector --- include/runtime/arena.h | 3 --- runtime/collect/collect.cpp | 17 +---------------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 464cc60f0..a9ef9fc75 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -150,9 +150,6 @@ class arena { // 127. #define REGISTER_ARENA(name, id) static thread_local arena name(id) -#define MEM_BLOCK_START(ptr) \ - ((char *)(((uintptr_t)(ptr)-1) & ~(BLOCK_SIZE - 1))) - #ifdef __MACH__ // // thread_local disabled for Apple diff --git a/runtime/collect/collect.cpp b/runtime/collect/collect.cpp index cc596d205..c439fdd0e 100644 --- a/runtime/collect/collect.cpp +++ b/runtime/collect/collect.cpp @@ -316,22 +316,7 @@ void kore_collect( if (collect_old || !previous_oldspace_alloc_ptr) { scan_ptr = oldspace_ptr(); } else { - if (MEM_BLOCK_START(previous_oldspace_alloc_ptr + 1) - == previous_oldspace_alloc_ptr) { - // this means that the previous oldspace allocation pointer points to an - // address that is megabyte-aligned. This can only happen if we have just - // filled up a block but have not yet allocated the next block in the - // sequence at the start of the collection cycle. This means that the - // allocation pointer is invalid and does not actually point to the next - // address that would have been allocated at, according to the logic of - // kore_arena_alloc, which will have allocated a fresh memory block and put - // the allocation at the start of it. Thus, we use arena::move_ptr with a size - // of zero to adjust and get the true address of the allocation. - scan_ptr - = arena::move_ptr(previous_oldspace_alloc_ptr, 0, *old_alloc_ptr()); - } else { - scan_ptr = previous_oldspace_alloc_ptr; - } + scan_ptr = previous_oldspace_alloc_ptr; } if (scan_ptr != *old_alloc_ptr()) { MEM_LOG("Evacuating old generation\n"); From e27ec1bb596118804ab077c4735170e358c6d5e0 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 3 Dec 2024 22:01:18 +0100 Subject: [PATCH 15/23] fix call to initialize_semispace() --- include/runtime/arena.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index a9ef9fc75..3f71445e4 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -204,7 +204,7 @@ inline void arena::arena_swap_and_clear() { // // The other semispace hasn't be initialized yet. // - void initialize_semispace(); + initialize_semispace(); } else arena_clear(); } From 6af3f6b3f76c185113066a846810a5abc777fc2c Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Wed, 4 Dec 2024 01:04:03 +0100 Subject: [PATCH 16/23] deleted dead code youngspace_size() and arena_size() --- include/runtime/alloc.h | 2 +- include/runtime/arena.h | 7 ------- runtime/lto/alloc.cpp | 4 ---- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/include/runtime/alloc.h b/include/runtime/alloc.h index d94ed0c5b..d2eeb1fc1 100644 --- a/include/runtime/alloc.h +++ b/include/runtime/alloc.h @@ -15,7 +15,7 @@ extern "C" { char youngspace_collection_id(void); char oldspace_collection_id(void); -size_t youngspace_size(void); + //size_t youngspace_size(void); // allocates exactly requested bytes into the young generation void *kore_alloc(size_t requested); diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 3f71445e4..ad7aade12 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -38,13 +38,6 @@ class arena { // This address is 0 if nothing has been allocated ever in that arena. char **arena_end_ptr() { return &allocation_ptr; } - // return the total number of allocatable bytes currently in the arena in its - // active semispace. - size_t arena_size() const { - update_num_blocks(); - return BLOCK_SIZE * std::max(num_blocks, num_collection_blocks); - } - // Clears the current allocation space by setting its start back to its first // block. It is used during garbage collection to effectively collect all of the // arena. diff --git a/runtime/lto/alloc.cpp b/runtime/lto/alloc.cpp index 0cd79a3f8..2928ad62f 100644 --- a/runtime/lto/alloc.cpp +++ b/runtime/lto/alloc.cpp @@ -39,10 +39,6 @@ char oldspace_collection_id() { return oldspace.get_arena_collection_semispace_id(); } -size_t youngspace_size(void) { - return youngspace.arena_size(); -} - void kore_alloc_swap(bool swap_old) { youngspace.arena_swap_and_clear(); if (swap_old) { From bfba1cacf035d635ee5b1da594e9c4b669dfb623 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Wed, 4 Dec 2024 03:58:51 +0100 Subject: [PATCH 17/23] replaced memory_block_header and associated functionality --- include/runtime/arena.h | 42 +++++++++++++++++++---------------------- runtime/alloc/arena.cpp | 15 ++++----------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index ad7aade12..c030f7e3c 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -28,10 +28,7 @@ class arena { // Returns the address of the first byte that belongs in the given arena. // Returns 0 if nothing has been allocated ever in that arena. - char *arena_start_ptr() const { - return current_addr_ptr ? current_addr_ptr + sizeof(memory_block_header) - : nullptr; - } + char *arena_start_ptr() const { return current_addr_ptr; } // Returns a pointer to a location holding the address of last allocated // byte in the given arena plus 1. @@ -40,7 +37,7 @@ class arena { // Clears the current allocation space by setting its start back to its first // block. It is used during garbage collection to effectively collect all of the - // arena. + // arena. Resets the tripwire. void arena_clear(); // Resizes the last allocation as long as the resize does not require a new @@ -90,15 +87,6 @@ class arena { static char get_arena_semispace_id_of_object(void *ptr); private: - union memory_block_header { - // - // Currently the header just holds the semispace id. But we need it to be a - // multiple of sizeof(char*) for alignment purposes so we add a dummy char*. - // - char semispace; - char *alignment_dummy; - }; - // // We update the number of 1MB blocks actually written to, only when we need this value, // or before a garbage collection rather than trying to determine when we write to a fresh block. @@ -114,13 +102,6 @@ class arena { } void initialize_semispace(); - - static memory_block_header *mem_block_header(void *ptr) { - uintptr_t address = reinterpret_cast(ptr); - return reinterpret_cast( - (address - 1) & ~(HYPERBLOCK_SIZE - 1)); - } - // // Current semispace where allocations are being made. // @@ -139,6 +120,18 @@ class arena { = 0; // notional number of BLOCK_SIZE blocks in collection semispace }; +inline char arena::get_arena_semispace_id_of_object(void *ptr) { + // + // We don't have to deal with the "1 past the end of block" case because + // a valid pointer will always point into our hyperblock - we will never return + // an allocation anywhere near the end of our hyperblock. + // + // Set the low bits to 1 to get the address of the last byte in the hyperblock. + // + uintptr_t end_address = reinterpret_cast(ptr) | (HYPERBLOCK_SIZE - 1); + return *reinterpret_cast(end_address); +} + // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to // 127. #define REGISTER_ARENA(name, id) static thread_local arena name(id) @@ -162,8 +155,11 @@ inline void *arena::kore_arena_alloc(size_t requested) { // collect when allowed. // time_for_collection = true; - tripwire = current_addr_ptr - + HYPERBLOCK_SIZE; // won't trigger again until arena swap + // + // We move the tripwire to 1 past the end of our hyperblock so that we have + // a well defined comparison that will always be false until the next arena swap. + // + tripwire = current_addr_ptr + HYPERBLOCK_SIZE; } void *result = allocation_ptr; allocation_ptr += requested; diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index fe0bfce45..94fdb85af 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -11,11 +11,6 @@ extern size_t const VAR_BLOCK_SIZE = BLOCK_SIZE; -__attribute__((always_inline)) char -arena::get_arena_semispace_id_of_object(void *ptr) { - return mem_block_header(ptr)->semispace; -} - #ifdef __MACH__ // // thread_local disabled for Apple @@ -53,13 +48,11 @@ void arena::initialize_semispace() { current_addr_ptr = reinterpret_cast( std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); // - // We put a memory_block_header at the beginning so we can identify the semispace a pointer belongs to - // id by masking off the low bits to access this memory_block_header. + // We put a semispace id in the last byte of the hyperblock so we can identify which semispace a pointer + // belongs to by setting the low bits to 1 to access this id. // - memory_block_header *header - = reinterpret_cast(current_addr_ptr); - header->semispace = allocation_semispace_id; - allocation_ptr = current_addr_ptr + sizeof(arena::memory_block_header); + current_addr_ptr[HYPERBLOCK_SIZE - 1] = allocation_semispace_id; + allocation_ptr = current_addr_ptr; // // We set the tripwire for this space so we get trigger a garbage collection when we pass BLOCK_SIZE of memory // allocated from this space. From 8e31c96b608fc01441f434ef82aa7a440a4236be Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Wed, 4 Dec 2024 20:29:07 +0100 Subject: [PATCH 18/23] fix formatting; clean code and comments --- include/runtime/arena.h | 24 +++++++++++------------- runtime/alloc/arena.cpp | 5 +++-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index c030f7e3c..bac3968a3 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -27,12 +27,12 @@ class arena { void *kore_arena_alloc(size_t requested); // Returns the address of the first byte that belongs in the given arena. - // Returns 0 if nothing has been allocated ever in that arena. + // Returns nullptr if nothing has been allocated ever in that arena. char *arena_start_ptr() const { return current_addr_ptr; } // Returns a pointer to a location holding the address of last allocated // byte in the given arena plus 1. - // This address is 0 if nothing has been allocated ever in that arena. + // This address is nullptr if nothing has been allocated ever in that arena. char **arena_end_ptr() { return &allocation_ptr; } // Clears the current allocation space by setting its start back to its first @@ -40,9 +40,8 @@ class arena { // arena. Resets the tripwire. void arena_clear(); - // Resizes the last allocation as long as the resize does not require a new - // block allocation. - // Returns the address of the byte following the last newlly allocated byte. + // Resizes the last allocation. + // Returns the address of the byte following the last newly allocated byte. void *arena_resize_last_alloc(ssize_t increase) { return (allocation_ptr += increase); } @@ -61,10 +60,8 @@ class arena { void arena_swap_and_clear(); // Given two pointers to objects allocated in the same arena, return the number - // of bytes they are separated by within the virtual block of memory represented - // by the blocks of that arena. This difference will include blocks containing - // sentinel bytes. Undefined behavior will result if the pointers belong to - // different arenas. + // of bytes they are apart. Undefined behavior will result if the pointers + // don't belong to the same arena static ssize_t ptr_diff(char *ptr1, char *ptr2) { return ptr1 - ptr2; } // Given a starting pointer to an address allocated in an arena and a size in @@ -74,11 +71,11 @@ class arena { // 1st argument: the starting pointer // 2nd argument: the size in bytes to add to the starting pointer // 3rd argument: the address of last allocated byte in the arena plus 1 - // Return value: the address allocated in the arena after size bytes from the - // starting pointer, or 0 if this is equal to the 3rd argument. + // Return value: starting pointer + size unless this points to unallocated space + // in which case nullptr is returned static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { char *next_ptr = ptr + size; - return (next_ptr == arena_end_ptr) ? 0 : next_ptr; + return (next_ptr == arena_end_ptr) ? nullptr : next_ptr; } // Returns the ID of the semispace where the given address was allocated. @@ -128,7 +125,8 @@ inline char arena::get_arena_semispace_id_of_object(void *ptr) { // // Set the low bits to 1 to get the address of the last byte in the hyperblock. // - uintptr_t end_address = reinterpret_cast(ptr) | (HYPERBLOCK_SIZE - 1); + uintptr_t end_address + = reinterpret_cast(ptr) | (HYPERBLOCK_SIZE - 1); return *reinterpret_cast(end_address); } diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 94fdb85af..c657dada4 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -41,14 +41,15 @@ void arena::initialize_semispace() { } // // We allocated 2 * HYPERBLOCK_SIZE worth of address space but we're only going to use 1, aligned on a - // HYPERBLOCK_SIZE boundry. This is so we can get the start of the hyperblock by masking any address within it. + // HYPERBLOCK_SIZE boundry. This is so we can get end of the hyperblock by setting the low bits of any + // address within the space to 1. // We don't worry about unused address space either side of our aligned address space because there will be no // memory mapped to it. // current_addr_ptr = reinterpret_cast( std::align(HYPERBLOCK_SIZE, HYPERBLOCK_SIZE, addr, request)); // - // We put a semispace id in the last byte of the hyperblock so we can identify which semispace a pointer + // We put a semispace id in the last byte of the hyperblock so we can identify which semispace an address // belongs to by setting the low bits to 1 to access this id. // current_addr_ptr[HYPERBLOCK_SIZE - 1] = allocation_semispace_id; From 9f2e3aa5d600adc065ae41feda004149550a370b Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Wed, 4 Dec 2024 20:51:53 +0100 Subject: [PATCH 19/23] fix formatting --- include/runtime/alloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/runtime/alloc.h b/include/runtime/alloc.h index d2eeb1fc1..4b10e4779 100644 --- a/include/runtime/alloc.h +++ b/include/runtime/alloc.h @@ -15,7 +15,7 @@ extern "C" { char youngspace_collection_id(void); char oldspace_collection_id(void); - //size_t youngspace_size(void); +//size_t youngspace_size(void); // allocates exactly requested bytes into the young generation void *kore_alloc(size_t requested); From 98b8404b4a96925870d92263bcd1735c305bc25b Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Thu, 5 Dec 2024 02:49:40 +0100 Subject: [PATCH 20/23] make arena_end_ptr() return char * --- include/runtime/arena.h | 6 +++--- runtime/collect/collect.cpp | 18 +++++++++--------- runtime/lto/alloc.cpp | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index bac3968a3..e9e29cff7 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -33,7 +33,7 @@ class arena { // Returns a pointer to a location holding the address of last allocated // byte in the given arena plus 1. // This address is nullptr if nothing has been allocated ever in that arena. - char **arena_end_ptr() { return &allocation_ptr; } + char *arena_end_ptr() { return allocation_ptr; } // Clears the current allocation space by setting its start back to its first // block. It is used during garbage collection to effectively collect all of the @@ -73,9 +73,9 @@ class arena { // 3rd argument: the address of last allocated byte in the arena plus 1 // Return value: starting pointer + size unless this points to unallocated space // in which case nullptr is returned - static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { + static char *move_ptr(char *ptr, size_t size, char const *end_ptr) { char *next_ptr = ptr + size; - return (next_ptr == arena_end_ptr) ? nullptr : next_ptr; + return (next_ptr == end_ptr) ? nullptr : next_ptr; } // Returns the ID of the semispace where the given address was allocated. diff --git a/runtime/collect/collect.cpp b/runtime/collect/collect.cpp index c439fdd0e..0c692453e 100644 --- a/runtime/collect/collect.cpp +++ b/runtime/collect/collect.cpp @@ -11,8 +11,8 @@ extern "C" { -char **young_alloc_ptr(void); -char **old_alloc_ptr(void); +char *young_alloc_ptr(void); +char *old_alloc_ptr(void); char *youngspace_ptr(void); char *oldspace_ptr(void); @@ -245,7 +245,7 @@ static void migrate_root(void *curr_block, layoutitem *args, unsigned i) { } } -static char *evacuate(char *scan_ptr, char **alloc_ptr) { +static char *evacuate(char *scan_ptr, char *alloc_ptr) { auto *curr_block = (block *)scan_ptr; uint64_t const hdr = curr_block->h.hdr; uint16_t layout_int = layout_hdr(hdr); @@ -255,7 +255,7 @@ static char *evacuate(char *scan_ptr, char **alloc_ptr) { migrate_child(curr_block, layout_data->args, i, false); } } - return arena::move_ptr(scan_ptr, get_size(hdr, layout_int), *alloc_ptr); + return arena::move_ptr(scan_ptr, get_size(hdr, layout_int), alloc_ptr); } // Contains the decision logic for collecting the old generation. @@ -293,7 +293,7 @@ void kore_collect( if (!last_alloc_ptr) { last_alloc_ptr = youngspace_ptr(); } - char *current_alloc_ptr = *young_alloc_ptr(); + char *current_alloc_ptr = young_alloc_ptr(); #endif kore_alloc_swap(collect_old); #ifdef GC_DBG @@ -301,13 +301,13 @@ void kore_collect( numBytesLiveAtCollection[i] = 0; } #endif - char *previous_oldspace_alloc_ptr = *old_alloc_ptr(); + char *previous_oldspace_alloc_ptr = old_alloc_ptr(); for (int i = 0; i < nroots; i++) { migrate_root(roots, type_info, i); } migrate_static_roots(); char *scan_ptr = youngspace_ptr(); - if (scan_ptr != *young_alloc_ptr()) { + if (scan_ptr != young_alloc_ptr()) { MEM_LOG("Evacuating young generation\n"); while (scan_ptr) { scan_ptr = evacuate(scan_ptr, young_alloc_ptr()); @@ -318,7 +318,7 @@ void kore_collect( } else { scan_ptr = previous_oldspace_alloc_ptr; } - if (scan_ptr != *old_alloc_ptr()) { + if (scan_ptr != old_alloc_ptr()) { MEM_LOG("Evacuating old generation\n"); while (scan_ptr) { scan_ptr = evacuate(scan_ptr, old_alloc_ptr()); @@ -329,7 +329,7 @@ void kore_collect( = arena::ptr_diff(current_alloc_ptr, last_alloc_ptr); assert(numBytesAllocedSinceLastCollection >= 0); fwrite(&numBytesAllocedSinceLastCollection, sizeof(ssize_t), 1, stderr); - last_alloc_ptr = *young_alloc_ptr(); + last_alloc_ptr = young_alloc_ptr(); fwrite( numBytesLiveAtCollection, sizeof(numBytesLiveAtCollection[0]), sizeof(numBytesLiveAtCollection) / sizeof(numBytesLiveAtCollection[0]), diff --git a/runtime/lto/alloc.cpp b/runtime/lto/alloc.cpp index 2928ad62f..629853980 100644 --- a/runtime/lto/alloc.cpp +++ b/runtime/lto/alloc.cpp @@ -23,11 +23,11 @@ char *oldspace_ptr() { return oldspace.arena_start_ptr(); } -char **young_alloc_ptr() { +char *young_alloc_ptr() { return youngspace.arena_end_ptr(); } -char **old_alloc_ptr() { +char *old_alloc_ptr() { return oldspace.arena_end_ptr(); } @@ -81,7 +81,7 @@ kore_resize_last_alloc(void *oldptr, size_t newrequest, size_t last_size) { newrequest = (newrequest + 7) & ~7; last_size = (last_size + 7) & ~7; - if (oldptr != *(youngspace.arena_end_ptr()) - last_size) { + if (oldptr != youngspace.arena_end_ptr() - last_size) { MEM_LOG( "May only reallocate last allocation. Tried to reallocate %p to %zd\n", oldptr, newrequest); From 9833230e430bfb154d4c282e6de1a9e646a7b918 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Thu, 5 Dec 2024 20:45:05 +0100 Subject: [PATCH 21/23] get rid of num_blocks and track touched bytes instead --- include/runtime/arena.h | 61 +++++++++++++++++++++-------------------- runtime/alloc/arena.cpp | 2 +- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index e9e29cff7..00fb2bd8c 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -12,6 +12,7 @@ extern "C" { size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; +size_t const margin = 1024 * 1024; // An arena can be used to allocate objects that can then be deallocated all at // once. @@ -84,20 +85,6 @@ class arena { static char get_arena_semispace_id_of_object(void *ptr); private: - // - // We update the number of 1MB blocks actually written to, only when we need this value, - // or before a garbage collection rather than trying to determine when we write to a fresh block. - // - void update_num_blocks() const { - // - // Calculate how many 1M blocks of the current arena we used. - // - size_t num_used_blocks - = (allocation_ptr - current_addr_ptr - 1) / BLOCK_SIZE + 1; - if (num_used_blocks > num_blocks) - num_blocks = num_used_blocks; - } - void initialize_semispace(); // // Current semispace where allocations are being made. @@ -105,16 +92,15 @@ class arena { char *current_addr_ptr; // pointer to start of current address space char *allocation_ptr; // next available location in current semispace char *tripwire; // allocating past this triggers slow allocation - mutable size_t - num_blocks; // notional number of BLOCK_SIZE blocks in current semispace + size_t nr_touched_bytes; // number of bytes that are touched char allocation_semispace_id; // id of current semispace // // Semispace where allocations will be made during and after garbage collect. + // We only track the start of the address space and the number of bytes + // that have been touched. // - char *collection_addr_ptr - = nullptr; // pointer to start of collection address space - size_t num_collection_blocks - = 0; // notional number of BLOCK_SIZE blocks in collection semispace + char *collection_addr_ptr = nullptr; + size_t nr_collection_touched_bytes = 0; }; inline char arena::get_arena_semispace_id_of_object(void *ptr) { @@ -172,27 +158,44 @@ inline void arena::arena_clear() { // We set the allocation pointer to the first available address. // allocation_ptr = arena_start_ptr(); + if (nr_touched_bytes >= get_gc_threshold() * BLOCK_SIZE) + tripwire = current_addr_ptr + nr_touched_bytes - margin; + else + tripwire = current_addr_ptr + nr_touched_bytes; // // If the number of blocks we've touched is >= threshold, we want to trigger // a garbage collection if we get within 1 block of the end of this area. // Otherwise we only want to generate a garbage collect if we allocate off the // end of this area. // - tripwire = current_addr_ptr - + (num_blocks - (num_blocks >= get_gc_threshold())) * BLOCK_SIZE; + //tripwire = current_addr_ptr + // + (num_blocks - (num_blocks >= get_gc_threshold())) * BLOCK_SIZE; } inline void arena::arena_swap_and_clear() { - update_num_blocks(); // so we save the correct number of touched blocks + // + // Calculate how many bytes we handled out from the current semispace + // before the swap. + // + size_t nr_allocated_bytes = allocation_ptr - current_addr_ptr; + // + // If this is greater than the number of bytes we touched on previous cycles, + // update the number of touched bytes. + // + if (nr_allocated_bytes > nr_touched_bytes) + nr_touched_bytes = nr_allocated_bytes; + // + // Swap the two key pieces of information with the other semispace and update the id. + // std::swap(current_addr_ptr, collection_addr_ptr); - std::swap(num_blocks, num_collection_blocks); + std::swap(nr_touched_bytes, nr_collection_touched_bytes); allocation_semispace_id = ~allocation_semispace_id; - if (current_addr_ptr == nullptr) { - // - // The other semispace hasn't be initialized yet. - // + // + // The semispace that is now current must either be initialized or cleared. + // + if (current_addr_ptr == nullptr) initialize_semispace(); - } else + else arena_clear(); } } diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index c657dada4..734aabc01 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -59,5 +59,5 @@ void arena::initialize_semispace() { // allocated from this space. // tripwire = current_addr_ptr + BLOCK_SIZE; - num_blocks = 1; + nr_touched_bytes = 0; } From 6d12050d02c50524ccc807a5c567975e1c5165d3 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Fri, 6 Dec 2024 01:17:22 +0100 Subject: [PATCH 22/23] move tripwire in 1 MB chucks --- include/runtime/arena.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 00fb2bd8c..08359ff46 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -92,7 +92,7 @@ class arena { char *current_addr_ptr; // pointer to start of current address space char *allocation_ptr; // next available location in current semispace char *tripwire; // allocating past this triggers slow allocation - size_t nr_touched_bytes; // number of bytes that are touched + size_t nr_touched_bytes; // number of bytes that are touched char allocation_semispace_id; // id of current semispace // // Semispace where allocations will be made during and after garbage collect. @@ -158,10 +158,12 @@ inline void arena::arena_clear() { // We set the allocation pointer to the first available address. // allocation_ptr = arena_start_ptr(); - if (nr_touched_bytes >= get_gc_threshold() * BLOCK_SIZE) - tripwire = current_addr_ptr + nr_touched_bytes - margin; + + size_t nr_touched_blocks = (nr_touched_bytes - 1) / BLOCK_SIZE + 1; + if (nr_touched_blocks >= get_gc_threshold() * BLOCK_SIZE) + tripwire = current_addr_ptr + (nr_touched_blocks - 1) * BLOCK_SIZE; else - tripwire = current_addr_ptr + nr_touched_bytes; + tripwire = current_addr_ptr + nr_touched_bytes * BLOCK_SIZE; // // If the number of blocks we've touched is >= threshold, we want to trigger // a garbage collection if we get within 1 block of the end of this area. @@ -198,5 +200,6 @@ inline void arena::arena_swap_and_clear() { else arena_clear(); } + } #endif // ARENA_H From dc9388fe7069439dcf0e26dd2ce853f204a5adb6 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Fri, 6 Dec 2024 03:17:12 +0100 Subject: [PATCH 23/23] revise tripwire calculations --- include/runtime/arena.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 08359ff46..601a78034 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -158,12 +158,16 @@ inline void arena::arena_clear() { // We set the allocation pointer to the first available address. // allocation_ptr = arena_start_ptr(); - - size_t nr_touched_blocks = (nr_touched_bytes - 1) / BLOCK_SIZE + 1; - if (nr_touched_blocks >= get_gc_threshold() * BLOCK_SIZE) - tripwire = current_addr_ptr + (nr_touched_blocks - 1) * BLOCK_SIZE; - else - tripwire = current_addr_ptr + nr_touched_bytes * BLOCK_SIZE; + // + // Put the tripwire pass the end of the blocks we've touched. + size_t nr_touched_blocks = (nr_touched_bytes == 0) ? 0 : (nr_touched_bytes - 1) / BLOCK_SIZE + 1; + tripwire = current_addr_ptr + nr_touched_bytes * BLOCK_SIZE; + // + // If we've touched enough blocks, pull back by margin, so we schedule a collection, + // before we need to touch virgin address space. + // + if (nr_touched_blocks >= get_gc_threshold()) + tripwire -= margin; // // If the number of blocks we've touched is >= threshold, we want to trigger // a garbage collection if we get within 1 block of the end of this area.