From f18f71c4db916a0691dfaec5357d8885fd1a03f3 Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Tue, 26 Nov 2024 12:33:57 -0600 Subject: [PATCH 1/5] make many global variables constant (#1167) This PR is a preparation to improve the thread safety of the llvm backend which attempts to reduce the number of global variables that need to be considered by marking as many of them as read only as possible. --- config/llvm_header.inc | 8 ++++---- include/runtime/types.h | 1 + lib/codegen/CreateStaticTerm.cpp | 8 ++++++++ lib/codegen/Decision.cpp | 2 ++ lib/codegen/EmitConfigParser.cpp | 13 +++++++++++++ lib/codegen/Metadata.cpp | 2 ++ 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/config/llvm_header.inc b/config/llvm_header.inc index b2359918e..5d665bb99 100644 --- a/config/llvm_header.inc +++ b/config/llvm_header.inc @@ -82,8 +82,8 @@ declare void @print_configuration(ptr, ptr) declare i64 @__gmpz_get_ui(ptr) -@exit_int_0 = global %mpz { i32 0, i32 0, ptr getelementptr inbounds ([0 x i64], ptr @exit_int_0_limbs, i32 0, i32 0) } -@exit_int_0_limbs = global [0 x i64] zeroinitializer +@exit_int_0 = constant %mpz { i32 0, i32 0, ptr getelementptr inbounds ([0 x i64], ptr @exit_int_0_limbs, i32 0, i32 0) } +@exit_int_0_limbs = constant [0 x i64] zeroinitializer define tailcc ptr @"eval_LblgetExitCode{SortGeneratedTopCell{}}"(ptr) { ret ptr @exit_int_0 @@ -110,8 +110,8 @@ declare ptr @hook_INT_add(ptr, ptr) declare ptr @evaluate_function_symbol(i32, ptr) declare ptr @get_terminated_string(ptr) -@fresh_int_1 = global %mpz { i32 1, i32 1, ptr getelementptr inbounds ([1 x i64], ptr @fresh_int_1_limbs, i32 0, i32 0) } -@fresh_int_1_limbs = global [1 x i64] [i64 1] +@fresh_int_1 = constant %mpz { i32 1, i32 1, ptr getelementptr inbounds ([1 x i64], ptr @fresh_int_1_limbs, i32 0, i32 0) } +@fresh_int_1_limbs = constant [1 x i64] [i64 1] define ptr @get_fresh_constant(ptr %sort, ptr %top) { entry: diff --git a/include/runtime/types.h b/include/runtime/types.h index fe230c6b2..de2b28ce1 100644 --- a/include/runtime/types.h +++ b/include/runtime/types.h @@ -1,6 +1,7 @@ #ifndef RUNTIME_TYPES_H #define RUNTIME_TYPES_H +#include #include #include diff --git a/lib/codegen/CreateStaticTerm.cpp b/lib/codegen/CreateStaticTerm.cpp index 7c69fa411..9234296f2 100644 --- a/lib/codegen/CreateStaticTerm.cpp +++ b/lib/codegen/CreateStaticTerm.cpp @@ -39,6 +39,9 @@ llvm::Constant *create_static_term::not_injection_case( llvm::Constant *block = module_->getOrInsertGlobal(kore_string.str(), block_type); auto *global_var = llvm::dyn_cast(block); + // this is technically not a constant because functions which return fresh constants + // will mutate a block in this circumstance. Probably best not to rely on this actually + // being mutable any other way. if (!global_var->hasInitializer()) { std::vector block_vals; @@ -151,6 +154,7 @@ create_static_term::create_token(value_type sort, std::string contents) { "int_" + contents, llvm::StructType::getTypeByName( module_->getContext(), int_wrapper_struct)); auto *global_var = llvm::dyn_cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { mpz_t value; char const *data_start @@ -163,6 +167,7 @@ create_static_term::create_token(value_type sort, std::string contents) { llvm::Constant *limbs = module_->getOrInsertGlobal( "int_" + contents + "_limbs", limbs_type); auto *limbs_var = llvm::dyn_cast(limbs); + limbs_var->setConstant(true); std::vector allocd_limbs; for (size_t i = 0; i < size; i++) { allocd_limbs.push_back(llvm::ConstantInt::get( @@ -205,6 +210,7 @@ create_static_term::create_token(value_type sort, std::string contents) { "float_" + contents, llvm::StructType::getTypeByName( module_->getContext(), float_wrapper_struct)); auto *global_var = llvm::dyn_cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { size_t prec = 0; size_t exp = 0; @@ -246,6 +252,7 @@ create_static_term::create_token(value_type sort, std::string contents) { llvm::Constant *limbs = module_->getOrInsertGlobal( "float_" + contents + "_limbs", limbs_type); auto *limbs_var = llvm::dyn_cast(limbs); + limbs_var->setConstant(true); std::vector allocd_limbs; for (size_t i = 0; i < size; i++) { allocd_limbs.push_back(llvm::ConstantInt::get( @@ -317,6 +324,7 @@ create_static_term::create_token(value_type sort, std::string contents) { llvm::Constant *global = module_->getOrInsertGlobal("token_" + escape(contents), string_type); auto *global_var = llvm::dyn_cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { llvm::StructType *block_header_type = llvm::StructType::getTypeByName( module_->getContext(), blockheader_struct); diff --git a/lib/codegen/Decision.cpp b/lib/codegen/Decision.cpp index da4c019b8..3ac8dd2d9 100644 --- a/lib/codegen/Decision.cpp +++ b/lib/codegen/Decision.cpp @@ -694,6 +694,7 @@ llvm::Constant *decision::string_literal(std::string const &str) { auto *global = module_->getOrInsertGlobal("str_lit_" + str, str_cst->getType()); auto *global_var = llvm::cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str_cst); } @@ -1091,6 +1092,7 @@ std::pair, llvm::BasicBlock *> step_function_header( auto *layout = module->getOrInsertGlobal( "layout_item_rule_" + std::to_string(ordinal), layout_arr->getType()); auto *global_var = llvm::cast(layout); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(layout_arr); } diff --git a/lib/codegen/EmitConfigParser.cpp b/lib/codegen/EmitConfigParser.cpp index 8ee777d53..f9a041051 100644 --- a/lib/codegen/EmitConfigParser.cpp +++ b/lib/codegen/EmitConfigParser.cpp @@ -55,6 +55,7 @@ static llvm::Constant *get_symbol_name_ptr( auto *global = module->getOrInsertGlobal( fmt::format("sym_name_{}", name), str->getType()); auto *global_var = llvm::dyn_cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str); } @@ -144,6 +145,7 @@ static void emit_data_table_for_symbol( auto *table_type = llvm::ArrayType::get(ty, syms.size()); auto *table = module->getOrInsertGlobal("table_" + name, table_type); auto *global_var = llvm::cast(table); + global_var->setConstant(true); init_debug_global( "table_" + name, get_array_debug_type( @@ -437,6 +439,7 @@ emit_get_tag_for_fresh_sort(kore_definition *definition, llvm::Module *module) { auto *global = module->getOrInsertGlobal("sort_name_" + name, str->getType()); auto *global_var = llvm::cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str); } @@ -504,6 +507,7 @@ static void emit_get_token(kore_definition *definition, llvm::Module *module) { auto *global = module->getOrInsertGlobal("sort_name_" + name, str->getType()); auto *global_var = llvm::dyn_cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str); } @@ -531,6 +535,7 @@ static void emit_get_token(kore_definition *definition, llvm::Module *module) { auto *str = llvm::ConstantDataArray::getString(ctx, "true", false); auto *global = module->getOrInsertGlobal("bool_true", str->getType()); auto *global_var = llvm::dyn_cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str); } @@ -625,6 +630,8 @@ static void emit_get_token(kore_definition *definition, llvm::Module *module) { string_type, block, {zero, zero32, zero32}, "", current_block); auto *block_size = module->getOrInsertGlobal( "VAR_BLOCK_SIZE", llvm::Type::getInt64Ty(ctx)); + auto *global_var = llvm::dyn_cast(block_size); + global_var->setConstant(true); auto *block_size_val = new llvm::LoadInst( llvm::Type::getInt64Ty(ctx), block_size, "", current_block); auto *block_alloc_size = llvm::BinaryOperator::Create( @@ -904,6 +911,7 @@ static void get_visitor( auto *global = module->getOrInsertGlobal( fmt::format("sort_name_{}", sort_name), str->getType()); auto *global_var = llvm::dyn_cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str); } @@ -1097,6 +1105,7 @@ static llvm::Constant *get_layout_data( auto *global = module->getOrInsertGlobal( "layout_item_" + std::to_string(layout), arr->getType()); auto *global_var = llvm::cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(arr); } @@ -1109,6 +1118,7 @@ static llvm::Constant *get_layout_data( name, llvm::StructType::getTypeByName(module->getContext(), layout_struct)); auto *global_var2 = llvm::cast(global2); + global_var2->setConstant(true); init_debug_global(name, get_forward_decl(layout_struct), global_var2); if (!global_var2->hasInitializer()) { global_var2->setInitializer(llvm::ConstantStruct::get( @@ -1201,6 +1211,7 @@ static void emit_sort_table_for_proof_trace_serialization( auto *subtable = module->getOrInsertGlobal( fmt::format("sort_tags_{}", ast_to_string(*symbol)), subtable_type); auto *subtable_var = llvm::dyn_cast(subtable); + subtable_var->setConstant(true); init_debug_global( "sort_tags_" + symbol->get_name(), get_array_debug_type( @@ -1246,6 +1257,7 @@ static void emit_sort_table(kore_definition *def, llvm::Module *mod) { auto *subtable = module->getOrInsertGlobal( fmt::format("sorts_{}", ast_to_string(*symbol)), subtable_type); auto *subtable_var = llvm::dyn_cast(subtable); + subtable_var->setConstant(true); init_debug_global( "sorts_" + symbol->get_name(), get_array_debug_type( @@ -1304,6 +1316,7 @@ static void emit_return_sort_table(kore_definition *def, llvm::Module *mod) { auto *sort_name = module->getOrInsertGlobal("sort_name_" + sort_str, str_type); auto *global_var = llvm::cast(sort_name); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str); } diff --git a/lib/codegen/Metadata.cpp b/lib/codegen/Metadata.cpp index 2c5395604..2965dcd51 100644 --- a/lib/codegen/Metadata.cpp +++ b/lib/codegen/Metadata.cpp @@ -24,6 +24,7 @@ void add_boolean_flag( auto *global = mod.getOrInsertGlobal(name, i1_ty); auto *global_var = llvm::cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(enabled_cst); @@ -44,6 +45,7 @@ void add_kompiled_dir_symbol( auto *global = mod.getOrInsertGlobal(kompiled_dir, str->getType()); auto *global_var = llvm::cast(global); + global_var->setConstant(true); if (!global_var->hasInitializer()) { global_var->setInitializer(str); From 0f72ec9a4ed1d6f138d64f17c0f85e1db7ab6a0b Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Tue, 26 Nov 2024 11:03:02 -0800 Subject: [PATCH 2/5] Make global/static variables thread_local; use mmap() for managing memory (#1161) Memory allocation is simplified by using mmap() to allocate a single 1TB block of addresses and relying on demand paging. Global and static variables are made thread_local and setThreadLocal(true) is used on such variables in the code generator. --------- Co-authored-by: Steven Eker Co-authored-by: Dwight Guth --- config/llvm_header.inc | 2 +- include/runtime/arena.h | 9 +- include/runtime/collect.h | 4 +- include/runtime/header.h | 8 +- lib/codegen/CreateTerm.cpp | 35 +++++++- lib/codegen/Decision.cpp | 21 ++++- runtime/alloc/arena.cpp | 100 +++++++++++++++-------- runtime/alloc/register_gc_roots_enum.cpp | 2 +- runtime/arithmetic/int.cpp | 4 +- runtime/collect/collect.cpp | 10 +-- runtime/collect/migrate_static_roots.cpp | 6 +- runtime/lto/alloc.cpp | 4 +- unittests/runtime-collections/lists.cpp | 8 ++ 13 files changed, 154 insertions(+), 59 deletions(-) diff --git a/config/llvm_header.inc b/config/llvm_header.inc index 5d665bb99..ec8d6917e 100644 --- a/config/llvm_header.inc +++ b/config/llvm_header.inc @@ -202,7 +202,7 @@ declare void @write_configuration_to_proof_trace(ptr, ptr, i1) @current_interval = thread_local global i64 0 @GC_THRESHOLD = thread_local global i64 @GC_THRESHOLD@ -@gc_roots = global [256 x ptr] zeroinitializer +@gc_roots = thread_local global [256 x ptr] zeroinitializer define i64 @get_gc_threshold() { %threshold = load i64, ptr @GC_THRESHOLD diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 1c32a0fc5..6248b243d 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -30,12 +30,19 @@ using memory_block_header = struct { // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to // 127. #define REGISTER_ARENA(name, id) \ - static struct arena name = {.allocation_semispace_id = (id)} + static thread_local struct arena name = {.allocation_semispace_id = (id)} #define MEM_BLOCK_START(ptr) \ ((char *)(((uintptr_t)(ptr)-1) & ~(BLOCK_SIZE - 1))) +#ifdef __MACH__ +// +// thread_local disabled for Apple +// extern bool time_for_collection; +#else +extern thread_local bool time_for_collection; +#endif size_t get_gc_threshold(); diff --git a/include/runtime/collect.h b/include/runtime/collect.h index a4ef7d2b0..1d448fcd4 100644 --- a/include/runtime/collect.h +++ b/include/runtime/collect.h @@ -26,8 +26,8 @@ using set_node = set::iterator::node_t; using set_impl = set::iterator::tree_t; extern "C" { -extern size_t numBytesLiveAtCollection[1 << AGE_WIDTH]; -extern bool collect_old; +extern thread_local size_t numBytesLiveAtCollection[1 << AGE_WIDTH]; +extern thread_local bool collect_old; size_t get_size(uint64_t, uint16_t); void migrate_static_roots(void); void migrate(block **block_ptr); diff --git a/include/runtime/header.h b/include/runtime/header.h index 000ec7cd2..d82cae004 100644 --- a/include/runtime/header.h +++ b/include/runtime/header.h @@ -47,8 +47,14 @@ size_t hash_k(block *); void k_hash(block *, void *); bool hash_enter(void); void hash_exit(void); - +#ifdef __MACH__ +// +// thread_local disabled for Apple +// extern bool gc_enabled; +#else +extern thread_local bool gc_enabled; +#endif } class k_elem { diff --git a/lib/codegen/CreateTerm.cpp b/lib/codegen/CreateTerm.cpp index 201c615f6..3bb71c900 100644 --- a/lib/codegen/CreateTerm.cpp +++ b/lib/codegen/CreateTerm.cpp @@ -782,10 +782,25 @@ llvm::Value *create_term::disable_gc() { llvm::Constant *global = module_->getOrInsertGlobal("gc_enabled", llvm::Type::getInt1Ty(ctx_)); auto *global_var = llvm::cast(global); +#ifdef __MACH__ + // + // thread_local disabled for Apple + // + /* + global_var->setThreadLocal(true); + llvm::IRBuilder b(current_block_); + auto *global_var_address = b.CreateThreadLocalAddress(global_var); + */ + auto *global_var_address = global_var; +#else + global_var->setThreadLocal(true); + auto *global_var_address = global_var; +#endif auto *old_val = new llvm::LoadInst( - llvm::Type::getInt1Ty(ctx_), global_var, "was_enabled", current_block_); + llvm::Type::getInt1Ty(ctx_), global_var_address, "was_enabled", + current_block_); new llvm::StoreInst( - llvm::ConstantInt::getFalse(ctx_), global_var, current_block_); + llvm::ConstantInt::getFalse(ctx_), global_var_address, current_block_); return old_val; } @@ -793,7 +808,21 @@ void create_term::enable_gc(llvm::Value *was_enabled) { llvm::Constant *global = module_->getOrInsertGlobal("gc_enabled", llvm::Type::getInt1Ty(ctx_)); auto *global_var = llvm::cast(global); - new llvm::StoreInst(was_enabled, global_var, current_block_); +#ifdef __MACH__ + // + // thread_local disabled for Apple + // + /* + global_var->setThreadLocal(true); + llvm::IRBuilder b(current_block_); + auto *global_var_address = b.CreateThreadLocalAddress(global_var); + */ + auto *global_var_address = global_var; +#else + global_var->setThreadLocal(true); + auto *global_var_address = global_var; +#endif + new llvm::StoreInst(was_enabled, global_var_address, current_block_); } // We use tailcc calling convention for apply_rule_* and eval_* functions to diff --git a/lib/codegen/Decision.cpp b/lib/codegen/Decision.cpp index 3ac8dd2d9..6ac04460a 100644 --- a/lib/codegen/Decision.cpp +++ b/lib/codegen/Decision.cpp @@ -5,6 +5,7 @@ #include "kllvm/codegen/ProofEvent.h" #include "kllvm/codegen/Util.h" +#include "llvm/IR/IRBuilder.h" #include #include #include @@ -1006,9 +1007,25 @@ std::pair, llvm::BasicBlock *> step_function_header( auto *collection = module->getOrInsertGlobal( "time_for_collection", llvm::Type::getInt1Ty(module->getContext())); + +#ifdef __MACH__ + // + // thread_local disabled for Apple + // + /* + llvm::cast(collection)->setThreadLocal(true); + llvm::IRBuilder b(check_collect); + auto *collection_address = b.CreateThreadLocalAddress(collection); + */ + auto *collection_address = collection; +#else + llvm::cast(collection)->setThreadLocal(true); + auto *collection_address = collection; +#endif + auto *is_collection = new llvm::LoadInst( - llvm::Type::getInt1Ty(module->getContext()), collection, "is_collection", - check_collect); + llvm::Type::getInt1Ty(module->getContext()), collection_address, + "is_collection", check_collect); set_debug_loc(is_collection); auto *collect = llvm::BasicBlock::Create( module->getContext(), "isCollect", block->getParent()); diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index dd0f6b2e6..0e1e4de15 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "runtime/alloc.h" #include "runtime/arena.h" @@ -47,36 +49,68 @@ get_arena_semispace_id_of_object(void *ptr) { return mem_block_header(ptr)->semispace; } -static void *first_superblock_ptr = nullptr; -static void *superblock_ptr = nullptr; -static char **next_superblock_ptr = nullptr; -static unsigned blocks_left = 0; +// +// We will reserve enough address space for 1 million 1MB blocks. Might want to increase this on a > 1TB server. +// +size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; +static thread_local void *hyperblock_ptr = nullptr; // only needed for munmap() static void *megabyte_malloc() { - if (blocks_left == 0) { - blocks_left = 15; - if (int result - = posix_memalign(&superblock_ptr, BLOCK_SIZE, BLOCK_SIZE * 15)) { - errno = result; - perror("posix_memalign"); - } - if (!first_superblock_ptr) { - first_superblock_ptr = superblock_ptr; - } - if (next_superblock_ptr) { - *next_superblock_ptr = (char *)superblock_ptr; + // + // Return pointer to a BLOCK_SIZE chunk of memory with BLOCK_SIZE alignment. + // + static thread_local char *currentblock_ptr + = nullptr; // char* rather than void* to permit pointer arithmetic + if (currentblock_ptr) { + // + // We expect an page fault due to not being able to map physical memory to this block or the + // process to be killed by the OOM killer long before we run off the end of our address space. + // + currentblock_ptr += BLOCK_SIZE; + } else { + // + // First call - need to reserve the address space. + // + size_t request = HYPERBLOCK_SIZE; + void *addr = mmap( + nullptr, // let OS choose the address + request, // Linux and MacOS both allow up to 64TB + PROT_READ | PROT_WRITE, // read, write but not execute + MAP_ANONYMOUS | MAP_PRIVATE + | MAP_NORESERVE, // allocate address space only + -1, // no file backing + 0); // no offset + if (addr == MAP_FAILED) { + perror("mmap()"); + abort(); } - auto *hdr = (memory_block_header *)superblock_ptr; - next_superblock_ptr = &hdr->next_superblock; - hdr->next_superblock = nullptr; + hyperblock_ptr = addr; + // + // We ask for one block worth of address space less than we allocated so alignment will always succeed. + // We don't worry about unused address space either side of our aligned address space because there will be no + // memory mapped to it. + // + currentblock_ptr = reinterpret_cast( + std::align(BLOCK_SIZE, HYPERBLOCK_SIZE - BLOCK_SIZE, addr, request)); } - blocks_left--; - void *result = superblock_ptr; - superblock_ptr = (char *)superblock_ptr + BLOCK_SIZE; - return result; + return currentblock_ptr; } +void free_all_memory() { + // + // Frees all memory that was demand paged into this address range. + // + munmap(hyperblock_ptr, HYPERBLOCK_SIZE); +} + +#ifdef __MACH__ +// +// thread_local disabled for Apple +// bool time_for_collection; +#else +thread_local bool time_for_collection; +#endif static void fresh_block(struct arena *arena) { char *next_block = nullptr; @@ -122,7 +156,14 @@ static void fresh_block(struct arena *arena) { BLOCK_SIZE - sizeof(memory_block_header)); } +#ifdef __MACH__ +// +// thread_local disabled for Apple +// bool gc_enabled = true; +#else +thread_local bool gc_enabled = true; +#endif __attribute__((noinline)) void * do_alloc_slow(size_t requested, struct arena *arena) { @@ -229,16 +270,3 @@ size_t arena_size(const struct arena *arena) { : arena->num_collection_blocks) * (BLOCK_SIZE - sizeof(memory_block_header)); } - -void free_all_memory() { - auto *superblock = (memory_block_header *)first_superblock_ptr; - while (superblock) { - auto *next_superblock = (memory_block_header *)superblock->next_superblock; - free(superblock); - superblock = next_superblock; - } - first_superblock_ptr = nullptr; - superblock_ptr = nullptr; - next_superblock_ptr = nullptr; - blocks_left = 0; -} diff --git a/runtime/alloc/register_gc_roots_enum.cpp b/runtime/alloc/register_gc_roots_enum.cpp index bbb4b2269..2c1a3165a 100644 --- a/runtime/alloc/register_gc_roots_enum.cpp +++ b/runtime/alloc/register_gc_roots_enum.cpp @@ -3,7 +3,7 @@ #include "runtime/collect.h" #include "runtime/header.h" -std::vector block_enumerators; +thread_local std::vector block_enumerators; void register_gc_roots_enumerator(BlockEnumerator f) { block_enumerators.push_back(f); diff --git a/runtime/arithmetic/int.cpp b/runtime/arithmetic/int.cpp index fcb4feec6..e76333fb6 100644 --- a/runtime/arithmetic/int.cpp +++ b/runtime/arithmetic/int.cpp @@ -373,8 +373,8 @@ void int_hash(mpz_t i, void *hasher) { } } -gmp_randstate_t kllvm_rand_state; -bool kllvm_rand_state_initialized = false; +thread_local gmp_randstate_t kllvm_rand_state; +thread_local bool kllvm_rand_state_initialized = false; SortK hook_INT_srand(SortInt seed) { if (!kllvm_rand_state_initialized) { diff --git a/runtime/collect/collect.cpp b/runtime/collect/collect.cpp index 31b8c4b77..b519bc15b 100644 --- a/runtime/collect/collect.cpp +++ b/runtime/collect/collect.cpp @@ -16,15 +16,15 @@ char **old_alloc_ptr(void); char *youngspace_ptr(void); char *oldspace_ptr(void); -static bool is_gc = false; -bool collect_old = false; +static thread_local bool is_gc = false; +bool thread_local collect_old = false; #ifndef GC_DBG -static uint8_t num_collection_only_young = 0; +static thread_local uint8_t num_collection_only_young = 0; #else -static char *last_alloc_ptr; +static thread_local char *last_alloc_ptr; #endif -size_t numBytesLiveAtCollection[1 << AGE_WIDTH]; +size_t thread_local numBytesLiveAtCollection[1 << AGE_WIDTH]; bool during_gc() { return is_gc; diff --git a/runtime/collect/migrate_static_roots.cpp b/runtime/collect/migrate_static_roots.cpp index d162f0bb8..3474e83ee 100644 --- a/runtime/collect/migrate_static_roots.cpp +++ b/runtime/collect/migrate_static_roots.cpp @@ -2,10 +2,10 @@ #include "runtime/collect.h" -extern std::vector block_enumerators; +extern thread_local std::vector block_enumerators; -extern gmp_randstate_t kllvm_rand_state; -extern bool kllvm_rand_state_initialized; +extern thread_local gmp_randstate_t kllvm_rand_state; +extern thread_local bool kllvm_rand_state_initialized; extern "C" { diff --git a/runtime/lto/alloc.cpp b/runtime/lto/alloc.cpp index 86fa11dfc..bcb2601e2 100644 --- a/runtime/lto/alloc.cpp +++ b/runtime/lto/alloc.cpp @@ -159,8 +159,8 @@ static inline void *kore_alloc_collection(kllvm::sort_category cat) { void *mem = kore_alloc(sizeof(blockheader) + sizeof(collection) + sizeof(uint64_t)); auto *hdr = (blockheader *)mem; - static std::string name = get_raw_symbol_name(cat) + "{}"; - static blockheader hdr_val + static thread_local std::string name = get_raw_symbol_name(cat) + "{}"; + static thread_local blockheader hdr_val = get_block_header_for_symbol(get_tag_for_symbol_name(name.c_str())); *hdr = hdr_val; auto *offset = (uint64_t *)(hdr + 1); diff --git a/unittests/runtime-collections/lists.cpp b/unittests/runtime-collections/lists.cpp index d4d2a20d9..1aa13a4e0 100644 --- a/unittests/runtime-collections/lists.cpp +++ b/unittests/runtime-collections/lists.cpp @@ -62,7 +62,15 @@ block D1 = {{1}}; block *DUMMY1 = &D1; } +#ifdef __MACH__ +// +// thread_local disabled for Apple +// bool gc_enabled; +#else +thread_local bool gc_enabled; +#endif + size_t get_gc_threshold() { return SIZE_MAX; } From 03f0fa67232b79649d508ec97781428250d5cbde Mon Sep 17 00:00:00 2001 From: gtrepta <50716988+gtrepta@users.noreply.github.com> Date: Wed, 27 Nov 2024 11:50:43 -0600 Subject: [PATCH 3/5] Don't make static tokens constant (#1169) Fixes a regression introduced in #1167 when mutable bytes are enabled. Hooked functions like `replaceAtBytes` that get called on static byte strings were causing segfaults because they were in read-only memory. --- lib/codegen/CreateStaticTerm.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/codegen/CreateStaticTerm.cpp b/lib/codegen/CreateStaticTerm.cpp index 9234296f2..029b32a23 100644 --- a/lib/codegen/CreateStaticTerm.cpp +++ b/lib/codegen/CreateStaticTerm.cpp @@ -324,7 +324,6 @@ create_static_term::create_token(value_type sort, std::string contents) { llvm::Constant *global = module_->getOrInsertGlobal("token_" + escape(contents), string_type); auto *global_var = llvm::dyn_cast(global); - global_var->setConstant(true); if (!global_var->hasInitializer()) { llvm::StructType *block_header_type = llvm::StructType::getTypeByName( module_->getContext(), blockheader_struct); From 3a0b23d66790f0049f3600ff9ad2e49004387756 Mon Sep 17 00:00:00 2001 From: Steven Eker Date: Wed, 27 Nov 2024 10:44:05 -0800 Subject: [PATCH 4/5] further refactoring allocation code (#1165) * memory_block_header now a struct member of class arena; next_superblock field deleted * global function free_all_memory() deleted * global variable hyperblock_ptr deleted * global functions make ptr_diff(), move_ptr(), get_arena_semispace_id_of_object(), mem_block_header() now static member functions of class arena --------- Co-authored-by: Dwight Guth --- include/runtime/arena.h | 194 ++++++++++++------------- runtime/alloc/arena.cpp | 175 +++++++++------------- runtime/collect/collect.cpp | 13 +- runtime/collect/migrate_collection.cpp | 4 +- runtime/lto/alloc.cpp | 39 +++-- 5 files changed, 192 insertions(+), 233 deletions(-) diff --git a/include/runtime/arena.h b/include/runtime/arena.h index 6248b243d..fc72155fb 100644 --- a/include/runtime/arena.h +++ b/include/runtime/arena.h @@ -10,27 +10,100 @@ extern "C" { // An arena can be used to allocate objects that can then be deallocated all at // once. -struct arena { - char *first_block; - char *block; - char *block_start; - char *block_end; - char *first_collection_block; - size_t num_blocks; - size_t num_collection_blocks; - char allocation_semispace_id; -}; - -using memory_block_header = struct { - char *next_block; - char *next_superblock; - char semispace; +class arena { +public: + arena(char id) + : allocation_semispace_id(id) { } + + // Allocates the requested number of bytes as a contiguous region and returns a + // pointer to the first allocated byte. + // If called with requested size greater than the maximun single allocation + // size, the space is allocated in a general (not garbage collected pool). + void *kore_arena_alloc(size_t requested); + + // Returns the address of the first byte that belongs in the given arena. + // Returns 0 if nothing has been allocated ever in that arena. + char *arena_start_ptr() const; + + // Returns a pointer to a location holding the address of last allocated + // byte in the given arena plus 1. + // This address is 0 if nothing has been allocated ever in that arena. + char **arena_end_ptr(); + + // return the total number of allocatable bytes currently in the arena in its + // active semispace. + size_t arena_size() const; + + // Clears the current allocation space by setting its start back to its first + // block. It is used during garbage collection to effectively collect all of the + // arena. + void arena_clear(); + + // Resizes the last allocation as long as the resize does not require a new + // block allocation. + // Returns the address of the byte following the last newlly allocated byte when + // the resize succeeds, returns 0 otherwise. + void *arena_resize_last_alloc(ssize_t increase); + + // Returns the given arena's current collection semispace ID. + // Each arena has 2 semispace IDs one equal to the arena ID and the other equal + // to the 1's complement of the arena ID. At any time one of these semispaces + // is used for allocation and the other is used for collection. + char get_arena_collection_semispace_id() const; + + // Exchanges the current allocation and collection semispaces and clears the new + // current allocation semispace by setting its start back to its first block. + // It is used before garbage collection. + void arena_swap_and_clear(); + + // Given two pointers to objects allocated in the same arena, return the number + // of bytes they are separated by within the virtual block of memory represented + // by the blocks of that arena. This difference will include blocks containing + // sentinel bytes. Undefined behavior will result if the pointers belong to + // different arenas. + static ssize_t ptr_diff(char *ptr1, char *ptr2); + + // Given a starting pointer to an address allocated in an arena and a size in + // bytes, this function returns a pointer to an address allocated in the + // same arena after size bytes from the starting pointer. + // + // 1st argument: the starting pointer + // 2nd argument: the size in bytes to add to the starting pointer + // 3rd argument: the address of last allocated byte in the arena plus 1 + // Return value: the address allocated in the arena after size bytes from the + // starting pointer, or 0 if this is equal to the 3rd argument. + static char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr); + + // Returns the ID of the semispace where the given address was allocated. + // The behavior is undefined if called with an address that has not been + // allocated within an arena. + static char get_arena_semispace_id_of_object(void *ptr); + +private: + struct memory_block_header { + char *next_block; + char semispace; + }; + + void fresh_block(); + static memory_block_header *mem_block_header(void *ptr); + + // helper function for `kore_arena_alloc`. Do not call directly. + void *do_alloc_slow(size_t requested); + + char *first_block; // beginning of first block + char *block; // where allocations are being made in current block + char *block_start; // start of current block + char *block_end; // 1 past end of current block + char *first_collection_block; // beginning of other semispace + size_t num_blocks; // number of blocks in current semispace + size_t num_collection_blocks; // number of blocks in other semispace + char allocation_semispace_id; // id of current semispace }; // Macro to define a new arena with the given ID. Supports IDs ranging from 0 to // 127. -#define REGISTER_ARENA(name, id) \ - static thread_local struct arena name = {.allocation_semispace_id = (id)} +#define REGISTER_ARENA(name, id) static thread_local arena name(id) #define MEM_BLOCK_START(ptr) \ ((char *)(((uintptr_t)(ptr)-1) & ~(BLOCK_SIZE - 1))) @@ -46,92 +119,17 @@ extern thread_local bool time_for_collection; size_t get_gc_threshold(); -// Resets the given arena. -void arena_reset(struct arena *); - -// Returns the given arena's current allocation semispace ID. -// Each arena has 2 semispace IDs one equal to the arena ID and the other equal -// to the 1's complement of the arena ID. At any time one of these semispaces -// is used for allocation and the other is used for collection. -char get_arena_allocation_semispace_id(const struct arena *); - -// Returns the given arena's current collection semispace ID. -// See above for details. -char get_arena_collection_semispace_id(const struct arena *); - -// Returns the ID of the semispace where the given address was allocated. -// The behavior is undefined if called with an address that has not been -// allocated within an arena. -char get_arena_semispace_id_of_object(void *); - -// helper function for `kore_arena_alloc`. Do not call directly. -void *do_alloc_slow(size_t, struct arena *); - -// Allocates the requested number of bytes as a contiguous region and returns a -// pointer to the first allocated byte. -// If called with requested size greater than the maximun single allocation -// size, the space is allocated in a general (not garbage collected pool). -inline void *kore_arena_alloc(struct arena *arena, size_t requested) { - if (arena->block + requested > arena->block_end) { - return do_alloc_slow(requested, arena); +inline void *arena::kore_arena_alloc(size_t requested) { + if (block + requested > block_end) { + return do_alloc_slow(requested); } - void *result = arena->block; - arena->block += requested; + void *result = block; + block += requested; MEM_LOG( "Allocation at %p (size %zd), next alloc at %p (if it fits)\n", result, - requested, arena->block); + requested, block); return result; } - -// Resizes the last allocation as long as the resize does not require a new -// block allocation. -// Returns the address of the byte following the last newlly allocated byte when -// the resize succeeds, returns 0 otherwise. -void *arena_resize_last_alloc(struct arena *, ssize_t); - -// Exchanges the current allocation and collection semispaces and clears the new -// current allocation semispace by setting its start back to its first block. -// It is used before garbage collection. -void arena_swap_and_clear(struct arena *); - -// Clears the current allocation space by setting its start back to its first -// block. It is used during garbage collection to effectively collect all of the -// arena. -void arena_clear(struct arena *); - -// Returns the address of the first byte that belongs in the given arena. -// Returns 0 if nothing has been allocated ever in that arena. -char *arena_start_ptr(const struct arena *); - -// Returns a pointer to a location holding the address of last allocated -// byte in the given arena plus 1. -// This address is 0 if nothing has been allocated ever in that arena. -char **arena_end_ptr(struct arena *); - -// Given a starting pointer to an address allocated in an arena and a size in -// bytes, this function returns a pointer to an address allocated in the -// same arena after size bytes from the starting pointer. -// -// 1st argument: the starting pointer -// 2nd argument: the size in bytes to add to the starting pointer -// 3rd argument: the address of last allocated byte in the arena plus 1 -// Return value: the address allocated in the arena after size bytes from the -// starting pointer, or 0 if this is equal to the 3rd argument. -char *move_ptr(char *, size_t, char const *); - -// Given two pointers to objects allocated in the same arena, return the number -// of bytes they are separated by within the virtual block of memory represented -// by the blocks of that arena. This difference will include blocks containing -// sentinel bytes. Undefined behavior will result if the pointers belong to -// different arenas. -ssize_t ptr_diff(char *, char *); - -// return the total number of allocatable bytes currently in the arena in its -// active semispace. -size_t arena_size(const struct arena *); - -// Deallocates all the memory allocated for registered arenas. -void free_all_memory(void); } #endif // ARENA_H diff --git a/runtime/alloc/arena.cpp b/runtime/alloc/arena.cpp index 0e1e4de15..c4384642e 100644 --- a/runtime/alloc/arena.cpp +++ b/runtime/alloc/arena.cpp @@ -12,40 +12,20 @@ extern size_t const VAR_BLOCK_SIZE = BLOCK_SIZE; -__attribute__((always_inline)) memory_block_header * -mem_block_header(void *ptr) { +__attribute__((always_inline)) arena::memory_block_header * +arena::mem_block_header(void *ptr) { // NOLINTNEXTLINE(*-reinterpret-cast) - return reinterpret_cast( + return reinterpret_cast( ((uintptr_t)(ptr)-1) & ~(BLOCK_SIZE - 1)); } -__attribute__((always_inline)) void arena_reset(struct arena *arena) { - char id = arena->allocation_semispace_id; - if (id < 0) { - id = ~arena->allocation_semispace_id; - } - arena->first_block = nullptr; - arena->block = nullptr; - arena->block_start = nullptr; - arena->block_end = nullptr; - arena->first_collection_block = nullptr; - arena->num_blocks = 0; - arena->num_collection_blocks = 0; - arena->allocation_semispace_id = id; -} - -__attribute__((always_inline)) char -get_arena_allocation_semispace_id(const struct arena *arena) { - return arena->allocation_semispace_id; -} - __attribute__((always_inline)) char -get_arena_collection_semispace_id(const struct arena *arena) { - return ~arena->allocation_semispace_id; +arena::get_arena_collection_semispace_id() const { + return ~allocation_semispace_id; } __attribute__((always_inline)) char -get_arena_semispace_id_of_object(void *ptr) { +arena::get_arena_semispace_id_of_object(void *ptr) { return mem_block_header(ptr)->semispace; } @@ -53,7 +33,6 @@ get_arena_semispace_id_of_object(void *ptr) { // We will reserve enough address space for 1 million 1MB blocks. Might want to increase this on a > 1TB server. // size_t const HYPERBLOCK_SIZE = (size_t)BLOCK_SIZE * 1024 * 1024; -static thread_local void *hyperblock_ptr = nullptr; // only needed for munmap() static void *megabyte_malloc() { // @@ -84,7 +63,6 @@ static void *megabyte_malloc() { perror("mmap()"); abort(); } - hyperblock_ptr = addr; // // We ask for one block worth of address space less than we allocated so alignment will always succeed. // We don't worry about unused address space either side of our aligned address space because there will be no @@ -96,13 +74,6 @@ static void *megabyte_malloc() { return currentblock_ptr; } -void free_all_memory() { - // - // Frees all memory that was demand paged into this address range. - // - munmap(hyperblock_ptr, HYPERBLOCK_SIZE); -} - #ifdef __MACH__ // // thread_local disabled for Apple @@ -112,48 +83,47 @@ bool time_for_collection; thread_local bool time_for_collection; #endif -static void fresh_block(struct arena *arena) { +void arena::fresh_block() { char *next_block = nullptr; - if (arena->block_start == nullptr) { + if (block_start == nullptr) { next_block = (char *)megabyte_malloc(); - arena->first_block = next_block; - auto *next_header = (memory_block_header *)next_block; + first_block = next_block; + auto *next_header = (arena::memory_block_header *)next_block; next_header->next_block = nullptr; - next_header->semispace = arena->allocation_semispace_id; - arena->num_blocks++; + next_header->semispace = allocation_semispace_id; + num_blocks++; } else { - next_block = *(char **)arena->block_start; - if (arena->block != arena->block_end) { - if (arena->block_end - arena->block == 8) { - *(uint64_t *)arena->block - = NOT_YOUNG_OBJECT_BIT; // 8 bit sentinel value + next_block = *(char **)block_start; + if (block != block_end) { + if (block_end - block == 8) { + *(uint64_t *)block = NOT_YOUNG_OBJECT_BIT; // 8 bit sentinel value } else { - *(uint64_t *)arena->block = arena->block_end - arena->block - - 8; // 16-bit or more sentinel value + *(uint64_t *)block + = block_end - block - 8; // 16-bit or more sentinel value } } if (!next_block) { MEM_LOG( "Allocating new block for the first time in arena %d\n", - arena->allocation_semispace_id); + allocation_semispace_id); next_block = (char *)megabyte_malloc(); - *(char **)arena->block_start = next_block; - auto *next_header = (memory_block_header *)next_block; + *(char **)block_start = next_block; + auto *next_header = (arena::memory_block_header *)next_block; next_header->next_block = nullptr; - next_header->semispace = arena->allocation_semispace_id; - arena->num_blocks++; + next_header->semispace = allocation_semispace_id; + num_blocks++; time_for_collection = true; } } - if (!*(char **)next_block && arena->num_blocks >= get_gc_threshold()) { + if (!*(char **)next_block && num_blocks >= get_gc_threshold()) { time_for_collection = true; } - arena->block = next_block + sizeof(memory_block_header); - arena->block_start = next_block; - arena->block_end = next_block + BLOCK_SIZE; + block = next_block + sizeof(arena::memory_block_header); + block_start = next_block; + block_end = next_block + BLOCK_SIZE; MEM_LOG( - "New block at %p (remaining %zd)\n", arena->block, - BLOCK_SIZE - sizeof(memory_block_header)); + "New block at %p (remaining %zd)\n", block, + BLOCK_SIZE - sizeof(arena::memory_block_header)); } #ifdef __MACH__ @@ -165,63 +135,59 @@ bool gc_enabled = true; thread_local bool gc_enabled = true; #endif -__attribute__((noinline)) void * -do_alloc_slow(size_t requested, struct arena *arena) { +__attribute__((noinline)) void *arena::do_alloc_slow(size_t requested) { MEM_LOG( - "Block at %p too small, %zd remaining but %zd needed\n", arena->block, - arena->block_end - arena->block, requested); - if (requested > BLOCK_SIZE - sizeof(memory_block_header)) { + "Block at %p too small, %zd remaining but %zd needed\n", block, + block_end - block, requested); + if (requested > BLOCK_SIZE - sizeof(arena::memory_block_header)) { return malloc(requested); } - fresh_block(arena); - void *result = arena->block; - arena->block += requested; + fresh_block(); + void *result = block; + block += requested; MEM_LOG( "Allocation at %p (size %zd), next alloc at %p (if it fits)\n", result, - requested, arena->block); + requested, block); return result; } __attribute__((always_inline)) void * -arena_resize_last_alloc(struct arena *arena, ssize_t increase) { - if (arena->block + increase <= arena->block_end) { - arena->block += increase; - return arena->block; +arena::arena_resize_last_alloc(ssize_t increase) { + if (block + increase <= block_end) { + block += increase; + return block; } return nullptr; } -__attribute__((always_inline)) void arena_swap_and_clear(struct arena *arena) { - char *tmp = arena->first_block; - arena->first_block = arena->first_collection_block; - arena->first_collection_block = tmp; - size_t tmp2 = arena->num_blocks; - arena->num_blocks = arena->num_collection_blocks; - arena->num_collection_blocks = tmp2; - arena->allocation_semispace_id = ~arena->allocation_semispace_id; - arena_clear(arena); +__attribute__((always_inline)) void arena::arena_swap_and_clear() { + char *tmp = first_block; + first_block = first_collection_block; + first_collection_block = tmp; + size_t tmp2 = num_blocks; + num_blocks = num_collection_blocks; + num_collection_blocks = tmp2; + allocation_semispace_id = ~allocation_semispace_id; + arena_clear(); } -__attribute__((always_inline)) void arena_clear(struct arena *arena) { - arena->block = arena->first_block - ? arena->first_block + sizeof(memory_block_header) - : nullptr; - arena->block_start = arena->first_block; - arena->block_end - = arena->first_block ? arena->first_block + BLOCK_SIZE : nullptr; +__attribute__((always_inline)) void arena::arena_clear() { + block = first_block ? first_block + sizeof(arena::memory_block_header) + : nullptr; + block_start = first_block; + block_end = first_block ? first_block + BLOCK_SIZE : nullptr; } -__attribute__((always_inline)) char * -arena_start_ptr(const struct arena *arena) { - return arena->first_block ? arena->first_block + sizeof(memory_block_header) - : nullptr; +__attribute__((always_inline)) char *arena::arena_start_ptr() const { + return first_block ? first_block + sizeof(arena::memory_block_header) + : nullptr; } -__attribute__((always_inline)) char **arena_end_ptr(struct arena *arena) { - return &arena->block; +__attribute__((always_inline)) char **arena::arena_end_ptr() { + return █ } -char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { +char *arena::move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { char *next_ptr = ptr + size; if (next_ptr == arena_end_ptr) { return nullptr; @@ -233,23 +199,23 @@ char *move_ptr(char *ptr, size_t size, char const *arena_end_ptr) { if (!next_block) { return nullptr; } - return next_block + sizeof(memory_block_header); + return next_block + sizeof(arena::memory_block_header); } -ssize_t ptr_diff(char *ptr1, char *ptr2) { +ssize_t arena::ptr_diff(char *ptr1, char *ptr2) { if (MEM_BLOCK_START(ptr1) == MEM_BLOCK_START(ptr2)) { return ptr1 - ptr2; } - memory_block_header *hdr = mem_block_header(ptr2); + arena::memory_block_header *hdr = mem_block_header(ptr2); ssize_t result = 0; while (hdr != mem_block_header(ptr1) && hdr->next_block) { if (ptr2) { result += ((char *)hdr + BLOCK_SIZE) - ptr2; ptr2 = nullptr; } else { - result += (BLOCK_SIZE - sizeof(memory_block_header)); + result += (BLOCK_SIZE - sizeof(arena::memory_block_header)); } - hdr = (memory_block_header *)hdr->next_block; + hdr = (arena::memory_block_header *)hdr->next_block; } if (hdr == mem_block_header(ptr1)) { result += ptr1 - (char *)(hdr + 1); @@ -264,9 +230,8 @@ ssize_t ptr_diff(char *ptr1, char *ptr2) { return -ptr_diff(ptr2, ptr1); } -size_t arena_size(const struct arena *arena) { - return (arena->num_blocks > arena->num_collection_blocks - ? arena->num_blocks - : arena->num_collection_blocks) - * (BLOCK_SIZE - sizeof(memory_block_header)); +size_t arena::arena_size() const { + return (num_blocks > num_collection_blocks ? num_blocks + : num_collection_blocks) + * (BLOCK_SIZE - sizeof(arena::memory_block_header)); } diff --git a/runtime/collect/collect.cpp b/runtime/collect/collect.cpp index b519bc15b..cc596d205 100644 --- a/runtime/collect/collect.cpp +++ b/runtime/collect/collect.cpp @@ -85,9 +85,9 @@ void migrate_once(block **block_ptr) { return; } if (youngspace_collection_id() - == get_arena_semispace_id_of_object((void *)curr_block) + == arena::get_arena_semispace_id_of_object((void *)curr_block) || oldspace_collection_id() - == get_arena_semispace_id_of_object((void *)curr_block)) { + == arena::get_arena_semispace_id_of_object((void *)curr_block)) { migrate(block_ptr); } } @@ -255,7 +255,7 @@ static char *evacuate(char *scan_ptr, char **alloc_ptr) { migrate_child(curr_block, layout_data->args, i, false); } } - return move_ptr(scan_ptr, get_size(hdr, layout_int), *alloc_ptr); + return arena::move_ptr(scan_ptr, get_size(hdr, layout_int), *alloc_ptr); } // Contains the decision logic for collecting the old generation. @@ -325,9 +325,10 @@ void kore_collect( // allocation pointer is invalid and does not actually point to the next // address that would have been allocated at, according to the logic of // kore_arena_alloc, which will have allocated a fresh memory block and put - // the allocation at the start of it. Thus, we use move_ptr with a size + // the allocation at the start of it. Thus, we use arena::move_ptr with a size // of zero to adjust and get the true address of the allocation. - scan_ptr = move_ptr(previous_oldspace_alloc_ptr, 0, *old_alloc_ptr()); + scan_ptr + = arena::move_ptr(previous_oldspace_alloc_ptr, 0, *old_alloc_ptr()); } else { scan_ptr = previous_oldspace_alloc_ptr; } @@ -340,7 +341,7 @@ void kore_collect( } #ifdef GC_DBG ssize_t numBytesAllocedSinceLastCollection - = ptr_diff(current_alloc_ptr, last_alloc_ptr); + = arena::ptr_diff(current_alloc_ptr, last_alloc_ptr); assert(numBytesAllocedSinceLastCollection >= 0); fwrite(&numBytesAllocedSinceLastCollection, sizeof(ssize_t), 1, stderr); last_alloc_ptr = *young_alloc_ptr(); diff --git a/runtime/collect/migrate_collection.cpp b/runtime/collect/migrate_collection.cpp index e2870de3e..c6e644e0e 100644 --- a/runtime/collect/migrate_collection.cpp +++ b/runtime/collect/migrate_collection.cpp @@ -7,9 +7,9 @@ void migrate_collection_node(void **node_ptr) { string *curr_block = STRUCT_BASE(string, data, *node_ptr); if (youngspace_collection_id() - != get_arena_semispace_id_of_object((void *)curr_block) + != arena::get_arena_semispace_id_of_object((void *)curr_block) && oldspace_collection_id() - != get_arena_semispace_id_of_object((void *)curr_block)) { + != arena::get_arena_semispace_id_of_object((void *)curr_block)) { return; } uint64_t const hdr = curr_block->h.hdr; diff --git a/runtime/lto/alloc.cpp b/runtime/lto/alloc.cpp index bcb2601e2..0cd79a3f8 100644 --- a/runtime/lto/alloc.cpp +++ b/runtime/lto/alloc.cpp @@ -16,47 +16,42 @@ REGISTER_ARENA(oldspace, OLDSPACE_ID); REGISTER_ARENA(alwaysgcspace, ALWAYSGCSPACE_ID); char *youngspace_ptr() { - return arena_start_ptr(&youngspace); + return youngspace.arena_start_ptr(); } char *oldspace_ptr() { - return arena_start_ptr(&oldspace); + return oldspace.arena_start_ptr(); } char **young_alloc_ptr() { - return arena_end_ptr(&youngspace); + return youngspace.arena_end_ptr(); } char **old_alloc_ptr() { - return arena_end_ptr(&oldspace); + return oldspace.arena_end_ptr(); } char youngspace_collection_id() { - return get_arena_collection_semispace_id(&youngspace); + return youngspace.get_arena_collection_semispace_id(); } char oldspace_collection_id() { - return get_arena_collection_semispace_id(&oldspace); + return oldspace.get_arena_collection_semispace_id(); } size_t youngspace_size(void) { - return arena_size(&youngspace); -} - -bool youngspace_almost_full(size_t threshold) { - char *next_block = *(char **)youngspace.block_start; - return !next_block; + return youngspace.arena_size(); } void kore_alloc_swap(bool swap_old) { - arena_swap_and_clear(&youngspace); + youngspace.arena_swap_and_clear(); if (swap_old) { - arena_swap_and_clear(&oldspace); + oldspace.arena_swap_and_clear(); } } void kore_clear() { - arena_clear(&alwaysgcspace); + alwaysgcspace.arena_clear(); } void set_kore_memory_functions_for_gmp() { @@ -64,25 +59,25 @@ void set_kore_memory_functions_for_gmp() { } __attribute__((always_inline)) void *kore_alloc(size_t requested) { - return kore_arena_alloc(&youngspace, requested); + return youngspace.kore_arena_alloc(requested); } __attribute__((always_inline)) void *kore_alloc_token(size_t requested) { size_t size = (requested + 7) & ~7; - return kore_arena_alloc(&youngspace, size < 16 ? 16 : size); + return youngspace.kore_arena_alloc(size < 16 ? 16 : size); } __attribute__((always_inline)) void *kore_alloc_old(size_t requested) { - return kore_arena_alloc(&oldspace, requested); + return oldspace.kore_arena_alloc(requested); } __attribute__((always_inline)) void *kore_alloc_token_old(size_t requested) { size_t size = (requested + 7) & ~7; - return kore_arena_alloc(&oldspace, size < 16 ? 16 : size); + return oldspace.kore_arena_alloc(size < 16 ? 16 : size); } __attribute__((always_inline)) void *kore_alloc_always_gc(size_t requested) { - return kore_arena_alloc(&alwaysgcspace, requested); + return alwaysgcspace.kore_arena_alloc(requested); } void * @@ -90,7 +85,7 @@ kore_resize_last_alloc(void *oldptr, size_t newrequest, size_t last_size) { newrequest = (newrequest + 7) & ~7; last_size = (last_size + 7) & ~7; - if (oldptr != *arena_end_ptr(&youngspace) - last_size) { + if (oldptr != *(youngspace.arena_end_ptr()) - last_size) { MEM_LOG( "May only reallocate last allocation. Tried to reallocate %p to %zd\n", oldptr, newrequest); @@ -98,7 +93,7 @@ kore_resize_last_alloc(void *oldptr, size_t newrequest, size_t last_size) { } ssize_t increase = newrequest - last_size; - if (arena_resize_last_alloc(&youngspace, increase)) { + if (youngspace.arena_resize_last_alloc(increase)) { return oldptr; } From 1d701f51b279613ba72ffbc66ea4ea5ee368cbb3 Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Mon, 2 Dec 2024 11:31:24 -0600 Subject: [PATCH 5/5] Minor optimization to search (#1170) In the final version of the ULM, we want to enforce that semantics are deterministic. We do this using the search feature and the `--execute-to-branch` option which treats all states with more than one successor as final states and does not rewrite them further. I tested this by running the ethereum test suite with this option and all the tests passed (with a few trivial changes to the semantics). A minor change resulted from this effort: if only a single state is active at a given time, we do not need to do the expensive equality check performed by `erase`. It is sufficient to simply clear the hash set. --- runtime/util/search.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/runtime/util/search.cpp b/runtime/util/search.cpp index 4c52f5abf..901db51aa 100644 --- a/runtime/util/search.cpp +++ b/runtime/util/search.cpp @@ -72,7 +72,11 @@ std::unordered_set take_search_steps( while (!states.empty() && depth != 0) { state = states.front(); states.pop_front(); - states_set.erase(state); + if (states.empty()) { + states_set.clear(); + } else { + states_set.erase(state); + } if (depth > 0) { depth--;