From cd2cdefdc4ebb654db96f2b465fb7aa1ab1ccdfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Mon, 30 Mar 2020 10:57:35 +0100 Subject: [PATCH 01/13] immix --- CMakeLists.txt | 1 + Makefile | 10 +- src/alloc.c | 97 +++ src/allocator.c | 453 ---------- src/allocator.h | 13 - src/gc.c | 2129 +++++++++++++++++++++++------------------------ src/gc.h | 262 ++++++ src/hl.h | 39 +- src/memory.c | 150 ++++ src/std/array.c | 3 +- src/std/types.c | 3 +- 11 files changed, 1568 insertions(+), 1592 deletions(-) create mode 100644 src/alloc.c delete mode 100644 src/allocator.c delete mode 100644 src/allocator.h create mode 100644 src/gc.h create mode 100644 src/memory.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c0666068..6aea12028 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,6 +94,7 @@ endif() add_library(libhl SHARED ${pcre_srcs} + src/alloc.c src/gc.c ${std_srcs} ) diff --git a/Makefile b/Makefile index 8d0ad0584..09b8b198f 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ INSTALL_DIR ?= $(PREFIX) LIBS=fmt sdl ssl openal ui uv mysql -CFLAGS = -Wall -O3 -I src -msse2 -mfpmath=sse -std=c11 -I include/pcre -I include/mikktspace -I include/minimp3 -D LIBHL_EXPORTS +#CFLAGS = -D GC_ENABLE_DEBUG -Wall -O0 -I src -msse2 -mfpmath=sse -std=c11 -I include/pcre -I include/mikktspace -I include/minimp3 -D LIBHL_EXPORTS -D HL_NO_THREADS +CFLAGS = -Wall -O3 -I src -msse2 -mfpmath=sse -std=c11 -I include/pcre -I include/mikktspace -I include/minimp3 -D LIBHL_EXPORTS -D HL_NO_THREADS LFLAGS = -L. -lhl LIBFLAGS = HLFLAGS = -ldl @@ -18,7 +19,7 @@ PCRE = include/pcre/pcre_chartables.o include/pcre/pcre_compile.o include/pcre/p include/pcre/pcre_newline.o include/pcre/pcre_string_utils.o include/pcre/pcre_tables.o include/pcre/pcre_xclass.o \ include/pcre/pcre16_ord2utf16.o include/pcre/pcre16_valid_utf16.o include/pcre/pcre_ucd.o -RUNTIME = src/gc.o +RUNTIME = src/alloc.o src/gc.o STD = src/std/array.o src/std/buffer.o src/std/bytes.o src/std/cast.o src/std/date.o src/std/error.o src/std/debug.o \ src/std/file.o src/std/fun.o src/std/maps.o src/std/math.o src/std/obj.o src/std/random.o src/std/regexp.o \ @@ -104,7 +105,7 @@ ifdef DEBUG CFLAGS += -g endif -all: libhl hl libs +all: libhl hl libs libstatic install: mkdir -p $(INSTALL_DIR) @@ -125,6 +126,9 @@ libs: $(LIBS) libhl: ${LIB} ${CC} -o libhl.$(LIBEXT) -m${MARCH} ${LIBFLAGS} -shared ${LIB} -lpthread -lm +libstatic: ${LIB} + ar -rcs libhl.a ${LIB} + hlc: ${BOOT} ${CC} ${CFLAGS} -o hlc ${BOOT} ${LFLAGS} diff --git a/src/alloc.c b/src/alloc.c new file mode 100644 index 000000000..2dba37a95 --- /dev/null +++ b/src/alloc.c @@ -0,0 +1,97 @@ +#include "hl.h" + +#ifdef HL_WIN +# include +#else +# include +# include +#endif + +struct hl_alloc_block { + int size; + hl_alloc_block *next; + unsigned char *p; +}; + +void hl_alloc_init( hl_alloc *a ) { + a->cur = NULL; +} + +void *hl_malloc( hl_alloc *a, int size ) { + hl_alloc_block *b = a->cur; + void *p; + if( !size ) return NULL; + size += hl_pad_size(size,&hlt_dyn); + if( b == NULL || b->size <= size ) { + int alloc = size < 4096-sizeof(hl_alloc_block) ? 4096-sizeof(hl_alloc_block) : size; + b = (hl_alloc_block *)malloc(sizeof(hl_alloc_block) + alloc); + if( b == NULL ) hl_fatal("Out of memory (malloc)"); + b->p = ((unsigned char*)b) + sizeof(hl_alloc_block); + b->size = alloc; + b->next = a->cur; + a->cur = b; + } + p = b->p; + b->p += size; + b->size -= size; + return p; +} + +void *hl_zalloc( hl_alloc *a, int size ) { + void *p = hl_malloc(a,size); + if( p ) memset(p, 0, size); + return p; +} + +void hl_free( hl_alloc *a ) { + hl_alloc_block *b = a->cur; + int_val prev = 0; + int size = 0; + while( b ) { + hl_alloc_block *n = b->next; + size = (int)(b->p + b->size - ((unsigned char*)b)); + prev = (int_val)b; + free(b); + b = n; + } + // check if our allocator was not part of the last free block + if( (int_val)a < prev || (int_val)a > prev+size ) + a->cur = NULL; +} + +HL_PRIM void *hl_alloc_executable_memory( int size ) { +#ifdef __APPLE__ +# ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +# endif +#endif +#if defined(HL_WIN) && defined(HL_64) + static char *jit_address = (char*)0x000076CA9F000000; + void *ptr; +retry_jit_alloc: + ptr = VirtualAlloc(jit_address,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE); + if( !ptr ) { + jit_address = (char*)(((int_val)jit_address)>>1); // fix for Win7 - will eventually reach NULL + goto retry_jit_alloc; + } + jit_address += size + ((-size) & (GC_PAGE_SIZE - 1)); + return ptr; +#elif defined(HL_WIN) + void *ptr = VirtualAlloc(NULL,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE); + return ptr; +#elif defined(HL_CONSOLE) + return NULL; +#else + void *p; + p = mmap(NULL,size,PROT_READ|PROT_WRITE|PROT_EXEC,(MAP_PRIVATE|MAP_ANONYMOUS),-1,0); + return p; +#endif +} + +HL_PRIM void hl_free_executable_memory( void *c, int size ) { +#if defined(HL_WIN) + VirtualFree(c,0,MEM_RELEASE); +#elif !defined(HL_CONSOLE) + munmap(c, size); +#endif +} diff --git a/src/allocator.c b/src/allocator.c deleted file mode 100644 index f15725d79..000000000 --- a/src/allocator.c +++ /dev/null @@ -1,453 +0,0 @@ -/* - * Copyright (C)2005-2020 Haxe Foundation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#include "hl.h" - -#ifdef HL_WIN -# include -static unsigned int __inline TRAILING_ONES( unsigned int x ) { - DWORD msb = 0; - if( _BitScanForward( &msb, ~x ) ) - return msb; - return 32; -} -static unsigned int __inline TRAILING_ZEROES( unsigned int x ) { - DWORD msb = 0; - if( _BitScanForward( &msb, x ) ) - return msb; - return 32; -} -#else -static inline unsigned int TRAILING_ONES( unsigned int x ) { - return (~x) ? __builtin_ctz(~x) : 32; -} -static inline unsigned int TRAILING_ZEROES( unsigned int x ) { - return x ? __builtin_ctz(x) : 32; -} -#endif - -#define GC_PARTITIONS 9 -#define GC_PART_BITS 4 -#define GC_FIXED_PARTS 5 -static const int GC_SBITS[GC_PARTITIONS] = {0,0,0,0,0, 3,6,14,22}; - -#ifdef HL_64 -static const int GC_SIZES[GC_PARTITIONS] = {8,16,24,32,40, 8,64,1<<14,1<<22}; -# define GC_ALIGN_BITS 3 -#else -static const int GC_SIZES[GC_PARTITIONS] = {4,8,12,16,20, 8,64,1<<14,1<<22}; -# define GC_ALIGN_BITS 2 -#endif - - -#define GC_ALL_PAGES (GC_PARTITIONS << PAGE_KIND_BITS) -#define GC_ALIGN (1 << GC_ALIGN_BITS) - -static gc_pheader *gc_pages[GC_ALL_PAGES] = {NULL}; -static int gc_free_blocks[GC_ALL_PAGES] = {0}; -static gc_pheader *gc_free_pages[GC_ALL_PAGES] = {NULL}; - - -static gc_pheader *gc_allocator_new_page( int pid, int block, int size, int kind, bool varsize ) { - // increase size based on previously allocated pages - if( block < 256 ) { - int num_pages = 0; - gc_pheader *ph = gc_pages[pid]; - while( ph ) { - num_pages++; - ph = ph->next_page; - } - while( num_pages > 8 && (size<<1) / block <= GC_PAGE_SIZE ) { - size <<= 1; - num_pages /= 3; - } - } - - int start_pos = 0; - int max_blocks = size / block; - - gc_pheader *ph = gc_alloc_page(size, kind, max_blocks); - gc_allocator_page_data *p = &ph->alloc; - - p->block_size = block; - p->max_blocks = max_blocks; - p->sizes = NULL; - if( p->max_blocks > GC_PAGE_SIZE ) - hl_fatal("Too many blocks for this page"); - if( varsize ) { - if( p->max_blocks <= 8 ) - p->sizes = (unsigned char*)&p->sizes_ref; - else { - p->sizes = ph->base + start_pos; - start_pos += p->max_blocks; - } - MZERO(p->sizes,p->max_blocks); - } - int m = start_pos % block; - if( m ) start_pos += block - m; - p->first_block = start_pos / block; - p->next_block = p->first_block; - p->free_blocks = p->max_blocks - p->first_block; - - ph->next_page = gc_pages[pid]; - gc_pages[pid] = ph; - - return ph; -} - -static void *gc_alloc_fixed( int part, int kind ) { - int pid = (part << PAGE_KIND_BITS) | kind; - gc_pheader *ph = gc_free_pages[pid]; - gc_allocator_page_data *p; - unsigned char *ptr; - while( ph ) { - p = &ph->alloc; - if( ph->bmp ) { - int next = p->next_block; - while( true ) { - unsigned int fetch_bits = ((unsigned int*)ph->bmp)[next >> 5]; - int ones = TRAILING_ONES(fetch_bits >> (next&31)); - next += ones; - if( (next&31) == 0 && ones ) { - if( next >= p->max_blocks ) { - p->next_block = next; - break; - } - continue; - } - p->next_block = next; - if( next >= p->max_blocks ) - break; - goto alloc_fixed; - } - } else if( p->next_block < p->max_blocks ) - break; - ph = ph->next_page; - } - if( ph == NULL ) - ph = gc_allocator_new_page(pid, GC_SIZES[part], GC_PAGE_SIZE, kind, false); -alloc_fixed: - p = &ph->alloc; - ptr = ph->base + p->next_block * p->block_size; -# ifdef GC_DEBUG - { - int i; - if( p->next_block < p->first_block || p->next_block >= p->max_blocks ) - hl_fatal("assert"); - if( ph->bmp && (ph->bmp[p->next_block>>3]&(1<<(p->next_block&7))) != 0 ) - hl_fatal("Alloc on marked bit"); - for(i=0;iblock_size;i++) - if( ptr[i] != 0xDD ) - hl_fatal("assert"); - } -# endif - p->next_block++; - gc_free_pages[pid] = ph; - return ptr; -} - -static void *gc_alloc_var( int part, int size, int kind ) { - int pid = (part << PAGE_KIND_BITS) | kind; - gc_pheader *ph = gc_free_pages[pid]; - gc_allocator_page_data *p; - unsigned char *ptr; - int nblocks = size >> GC_SBITS[part]; - int max_free = gc_free_blocks[pid]; -loop: - while( ph ) { - p = &ph->alloc; - if( ph->bmp ) { - int next, avail = 0; - if( p->free_blocks >= nblocks ) { - p->next_block = p->first_block; - p->free_blocks = 0; - } - next = p->next_block; - if( next + nblocks > p->max_blocks ) - goto skip; - while( true ) { - int fid = next >> 5; - unsigned int fetch_bits = ((unsigned int*)ph->bmp)[fid]; - int bits; -resume: - bits = TRAILING_ONES(fetch_bits >> (next&31)); - if( bits ) { - if( avail > p->free_blocks ) p->free_blocks = avail; - avail = 0; - next += bits - 1; - if( next >= p->max_blocks ) { - p->next_block = next; - ph = ph->next_page; - goto loop; - } - if( p->sizes[next] == 0 ) hl_fatal("assert"); - next += p->sizes[next]; - if( next + nblocks > p->max_blocks ) { - p->next_block = next; - ph = ph->next_page; - goto loop; - } - if( (next>>5) != fid ) - continue; - goto resume; - } - bits = TRAILING_ZEROES( (next & 31) ? (fetch_bits >> (next&31)) | (1<<(32-(next&31))) : fetch_bits ); - avail += bits; - next += bits; - if( next > p->max_blocks ) { - avail -= next - p->max_blocks; - next = p->max_blocks; - if( avail < nblocks ) break; - } - if( avail >= nblocks ) { - p->next_block = next - avail; - goto alloc_var; - } - if( next & 31 ) goto resume; - } - if( avail > p->free_blocks ) p->free_blocks = avail; - p->next_block = next; - } else if( p->next_block + nblocks <= p->max_blocks ) - break; -skip: - if( p->free_blocks > max_free ) - max_free = p->free_blocks; - ph = ph->next_page; - if( ph == NULL && max_free >= nblocks ) { - max_free = 0; - ph = gc_pages[pid]; - } - } - if( ph == NULL ) { - int psize = GC_PAGE_SIZE; - while( psize < size + 1024 ) - psize <<= 1; - ph = gc_allocator_new_page(pid, GC_SIZES[part], psize, kind, true); - } -alloc_var: - p = &ph->alloc; - ptr = ph->base + p->next_block * p->block_size; -# ifdef GC_DEBUG - { - int i; - if( p->next_block < p->first_block || p->next_block + nblocks > p->max_blocks ) - hl_fatal("assert"); - for(i=0;ibmp ) { - int bid = p->next_block; -# ifdef GC_DEBUG - int i; - for(i=0;ibmp[bid>>3]&(1<<(bid&7))) != 0 ) hl_fatal("Alloc on marked block"); - bid++; - } - bid = p->next_block; -# endif - ph->bmp[bid>>3] |= 1<<(bid&7); - } else { - p->free_blocks = p->max_blocks - (p->next_block + nblocks); - } - if( nblocks > 1 ) MZERO(p->sizes + p->next_block, nblocks); - p->sizes[p->next_block] = (unsigned char)nblocks; - p->next_block += nblocks; - gc_free_pages[pid] = ph; - gc_free_blocks[pid] = max_free; - return ptr; -} - -static void *gc_allocator_alloc( int *size, int page_kind ) { - int sz = *size; - sz += (-sz) & (GC_ALIGN - 1); - if( sz <= GC_SIZES[GC_FIXED_PARTS-1] && page_kind != MEM_KIND_FINALIZER ) { - int part = (sz >> GC_ALIGN_BITS) - 1; - *size = GC_SIZES[part]; - return gc_alloc_fixed(part, page_kind); - } - int p; - for(p=GC_FIXED_PARTS;p>8 ) { - if( memcmp(p,ZEROMEM,256) ) return false; - p += 256; - size -= 256; - } - return memcmp(p,ZEROMEM,size) == 0; -} - -static void gc_flush_empty_pages() { - int i; - for(i=0;ialloc; - gc_pheader *next = ph->next_page; - if( ph->bmp && is_zero(ph->bmp+(p->first_block>>3),((p->max_blocks+7)>>3) - (p->first_block>>3)) ) { - if( prev ) - prev->next_page = next; - else - gc_pages[i] = next; - if( gc_free_pages[i] == ph ) - gc_free_pages[i] = next; - gc_free_page(ph, p->max_blocks); - } else - prev = ph; - ph = next; - } - } -} - -#ifdef GC_DEBUG -static void gc_clear_unmarked_mem() { - int i; - for(i=0;ialloc; - for(bid=p->first_block;bidmax_blocks;bid++) { - if( p->sizes && !p->sizes[bid] ) continue; - int size = p->sizes ? p->sizes[bid] * p->block_size : p->block_size; - unsigned char *ptr = ph->base + bid * p->block_size; - if( bid * p->block_size + size > ph->page_size ) hl_fatal("invalid block size"); -# ifdef GC_MEMCHK - int_val eob = *(int_val*)(ptr + size - HL_WSIZE); -# ifdef HL_64 - if( eob != 0xEEEEEEEEEEEEEEEE && eob != 0xDDDDDDDDDDDDDDDD ) -# else - if( eob != 0xEEEEEEEE && eob != 0xDDDDDDDD ) -# endif - hl_fatal("Block written out of bounds"); -# endif - if( (ph->bmp[bid>>3] & (1<<(bid&7))) == 0 ) { - memset(ptr,0xDD,size); - if( p->sizes ) p->sizes[bid] = 0; - } - } - ph = ph->next_page; - } - } -} -#endif - -static void gc_call_finalizers(){ - int i; - for(i=MEM_KIND_FINALIZER;ialloc; - for(bid=p->first_block;bidmax_blocks;bid++) { - int size = p->sizes[bid]; - if( !size ) continue; - if( (ph->bmp[bid>>3] & (1<<(bid&7))) == 0 ) { - unsigned char *ptr = ph->base + bid * p->block_size; - void *finalizer = *(void**)ptr; - p->sizes[bid] = 0; - if( finalizer ) - ((void(*)(void *))finalizer)(ptr); -# ifdef GC_DEBUG - memset(ptr,0xDD,size*p->block_size); -# endif - } - } - ph = ph->next_page; - } - } -} - -static void gc_allocator_before_mark( unsigned char *mark_cur ) { - int pid; - for(pid=0;pidbmp = mark_cur; - p->alloc.next_block = p->alloc.first_block; - p->alloc.free_blocks = 0; - mark_cur += (p->alloc.max_blocks + 7) >> 3; - p = p->next_page; - } - } -} - -#define gc_allocator_fast_block_size(page,block) \ - (page->alloc.sizes ? page->alloc.sizes[(int)(((unsigned char*)(block)) - page->base) / page->alloc.block_size] * page->alloc.block_size : page->alloc.block_size) - -static void gc_allocator_init() { - if( TRAILING_ONES(0x080003FF) != 10 || TRAILING_ONES(0) != 0 || TRAILING_ONES(0xFFFFFFFF) != 32 ) - hl_fatal("Invalid builtin tl1"); - if( TRAILING_ZEROES((unsigned)~0x080003FF) != 10 || TRAILING_ZEROES(0) != 32 || TRAILING_ZEROES(0xFFFFFFFF) != 0 ) - hl_fatal("Invalid builtin tl0"); -} - -static int gc_allocator_get_block_id( gc_pheader *page, void *block ) { - int offset = (int)((unsigned char*)block - page->base); - if( offset%page->alloc.block_size != 0 ) - return -1; - int bid = offset / page->alloc.block_size; - if( page->alloc.sizes && page->alloc.sizes[bid] == 0 ) return -1; - return bid; -} - -#ifdef GC_INTERIOR_POINTERS -static int gc_allocator_get_block_interior( gc_pheader *page, void **block ) { - int offset = (int)((unsigned char*)*block - page->base); - int bid = offset / page->alloc.block_size; - if( page->alloc.sizes ) { - while( page->alloc.sizes[bid] == 0 ) { - if( bid == page->alloc.first_block ) return -1; - bid--; - } - } - *block = page->base + bid * page->alloc.block_size; - return bid; -} -#endif - -static void gc_allocator_after_mark() { - gc_call_finalizers(); -# ifdef GC_DEBUG - gc_clear_unmarked_mem(); -# endif - gc_flush_empty_pages(); -} - - - diff --git a/src/allocator.h b/src/allocator.h deleted file mode 100644 index 30c60380c..000000000 --- a/src/allocator.h +++ /dev/null @@ -1,13 +0,0 @@ - -typedef struct { - int block_size; - int max_blocks; - int first_block; - // mutable - int next_block; - int free_blocks; - unsigned char *sizes; - int sizes_ref; - int sizes_ref2; -} gc_allocator_page_data; - diff --git a/src/gc.c b/src/gc.c index 70d4bc95d..0ab374167 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1,1105 +1,1024 @@ -/* - * Copyright (C)2005-2016 Haxe Foundation - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#include "hl.h" -#ifdef HL_WIN -# include -#else -# include -# include -#endif - -#define MZERO(ptr,size) memset(ptr,0,size) - -// GC - -#define GC_PAGE_BITS 16 -#define GC_PAGE_SIZE (1 << GC_PAGE_BITS) - -#ifndef HL_64 -# define gc_hash(ptr) ((unsigned int)(ptr)) -# define GC_LEVEL0_BITS 8 -# define GC_LEVEL1_BITS 8 -#else -# define GC_LEVEL0_BITS 10 -# define GC_LEVEL1_BITS 10 - -// we currently discard the higher bits -// we should instead have some special handling for them -// in x86-64 user space grows up to 0x8000-00000000 (16 bits base + 31 bits page id) - -#ifdef HL_WIN -# define gc_hash(ptr) ((int_val)(ptr)&0x0000000FFFFFFFFF) -#else -// Linux gives addresses using the following patterns (X=any,Y=small value - can be 0): -// 0x0000000YXXX0000 -// 0x0007FY0YXXX0000 -static int_val gc_hash( void *ptr ) { - int_val v = (int_val)ptr; - return (v ^ ((v >> 33) << 28)) & 0x0000000FFFFFFFFF; -} -#endif - -#endif - -#define GC_MASK_BITS 16 -#define GC_GET_LEVEL1(ptr) hl_gc_page_map[gc_hash(ptr)>>(GC_MASK_BITS+GC_LEVEL1_BITS)] -#define GC_GET_PAGE(ptr) GC_GET_LEVEL1(ptr)[(gc_hash(ptr)>>GC_MASK_BITS)&GC_LEVEL1_MASK] -#define GC_LEVEL1_MASK ((1 << GC_LEVEL1_BITS) - 1) - -#define PAGE_KIND_BITS 2 -#define PAGE_KIND_MASK ((1 << PAGE_KIND_BITS) - 1) - -#if defined(HL_DEBUG) && !defined(HL_CONSOLE) -# define GC_DEBUG -# define GC_MEMCHK -#endif - -#if defined(HL_NX) -# define GC_INTERIOR_POINTERS -#endif - -#define out_of_memory(reason) hl_fatal("Out of Memory (" reason ")") - -typedef struct _gc_pheader gc_pheader; - -#if 0 -typedef void* gc_allocator_page_data; - -// Initialize the allocator -void gc_allocator_init(); - -// Get the block size within the given page. The block validity has already been checked. -int gc_allocator_fast_block_size( gc_pheader *page, void *block ); - -// Get the block id within the given page, or -1 if it's an invalid ptr. The block is already checked within page bounds -int gc_allocator_get_block_id( gc_pheader *page, void *block ); - -// Same as get_block_id but handles interior pointers and modify the block value -int gc_allocator_get_block_id_interior( gc_pheader *page, void **block ); - -// Called before marking starts: should update each page "bmp" with mark_bits -void gc_allocator_before_mark( unsigned char *mark_bits ); - -// Called when marking ends: should call finalizers, sweep unused blocks and free empty pages -void gc_allocator_after_mark(); - -// Allocate a block with given size using the specified page kind. -// Returns NULL if no block could be allocated -// Sets size to really allocated size (could be larger) -// Sets size to -1 if allocation refused (required size is invalid) -void *gc_allocator_alloc( int *size, int page_kind ); - -#else -# include "allocator.h" -#endif - -struct _gc_pheader { - // const - unsigned char *base; - unsigned char *bmp; - int page_size; - int page_kind; - gc_allocator_page_data alloc; - gc_pheader *next_page; -#ifdef GC_DEBUG - int page_id; -#endif -}; - -#ifdef HL_64 -# define INPAGE(ptr,page) ((unsigned char*)(ptr) >= (page)->base && (unsigned char*)(ptr) < (page)->base + (page)->page_size) -#else -# define INPAGE(ptr,page) true -#endif - -#define GC_PROFILE 1 -#define GC_DUMP_MEM 2 -#define GC_NO_THREADS 4 -#define GC_FORCE_MAJOR 8 - -static int gc_flags = 0; -static gc_pheader *gc_level1_null[1<gc_regs); - t->stack_cur = &t; -} - -#ifndef HL_THREADS -# define gc_global_lock(_) -#else -static void gc_global_lock( bool lock ) { - hl_thread_info *t = current_thread; - bool mt = (gc_flags & GC_NO_THREADS) == 0; - if( !t && gc_threads.count == 0 ) return; - if( lock ) { - if( !t ) - hl_fatal("Can't lock GC in unregistered thread"); - if( mt ) gc_save_context(t); - t->gc_blocking++; - if( mt ) hl_mutex_acquire(gc_threads.global_lock); - } else { - t->gc_blocking--; - if( mt ) hl_mutex_release(gc_threads.global_lock); - } -} -#endif - -HL_PRIM void hl_add_root( void *r ) { - gc_global_lock(true); - if( gc_roots_count == gc_roots_max ) { - int nroots = gc_roots_max ? (gc_roots_max << 1) : 16; - void ***roots = (void***)malloc(sizeof(void*)*nroots); - memcpy(roots,gc_roots,sizeof(void*)*gc_roots_count); - free(gc_roots); - gc_roots = roots; - gc_roots_max = nroots; - } - gc_roots[gc_roots_count++] = (void**)r; - gc_global_lock(false); -} - -HL_PRIM void hl_remove_root( void *v ) { - int i; - gc_global_lock(true); - for(i=gc_roots_count-1;i>=0;i--) - if( gc_roots[i] == (void**)v ) { - gc_roots_count--; - memmove(gc_roots + i, gc_roots + (i+1), (gc_roots_count - i) * sizeof(void*)); - break; - } - gc_global_lock(false); -} - -HL_PRIM gc_pheader *hl_gc_get_page( void *v ) { - gc_pheader *page = GC_GET_PAGE(v); - if( page && !INPAGE(v,page) ) - page = NULL; - return page; -} - -// ------------------------- THREADS ---------------------------------------------------------- - -HL_API int hl_thread_id(); - -HL_API void hl_register_thread( void *stack_top ) { - if( hl_get_thread() ) - hl_fatal("Thread already registered"); - - hl_thread_info *t = (hl_thread_info*)malloc(sizeof(hl_thread_info)); - memset(t, 0, sizeof(hl_thread_info)); - t->thread_id = hl_thread_id(); - t->stack_top = stack_top; - t->flags = HL_TRACK_MASK << HL_TREAD_TRACK_SHIFT; - current_thread = t; - hl_add_root(&t->exc_value); - hl_add_root(&t->exc_handler); - - gc_global_lock(true); - hl_thread_info **all = (hl_thread_info**)malloc(sizeof(void*) * (gc_threads.count + 1)); - memcpy(all,gc_threads.threads,sizeof(void*)*gc_threads.count); - gc_threads.threads = all; - all[gc_threads.count++] = t; - gc_global_lock(false); -} - -HL_API void hl_unregister_thread() { - int i; - hl_thread_info *t = hl_get_thread(); - if( !t ) - hl_fatal("Thread not registered"); - hl_remove_root(&t->exc_value); - hl_remove_root(&t->exc_handler); - gc_global_lock(true); - for(i=0;igc_blocking == 0 ) {}; // spinwait - } - } else { - // releasing global lock will release all threads - gc_threads.stopping_world = false; - } -# else - if( b ) gc_save_context(current_thread); -# endif -} - -// ------------------------- ALLOCATOR ---------------------------------------------------------- - -#ifdef GC_DEBUG -static int PAGE_ID = 0; -#endif - -HL_API void hl_gc_dump_memory( const char *filename ); -static void gc_major( void ); - -static void *gc_will_collide( void *p, int size ) { -# ifdef HL_64 - int i; - for(i=0;i>GC_MASK_BITS;i++) { - void *ptr = (unsigned char*)p + (i<= (8 << 20) ) { - gc_global_lock(false); - hl_error("Failed to alloc %d KB",size>>10); - } - if( gc_flags & GC_DUMP_MEM ) hl_gc_dump_memory("hlmemory.dump"); - out_of_memory("pages"); - } - - gc_pheader *p = gc_free_pheaders; - if( !p ) { - // alloc pages by chunks so we get good memory locality - int i, count = 100; - gc_pheader *head = (gc_pheader*)malloc(sizeof(gc_pheader)*count); - p = head; - for(i=1;inext_page = head + i; - p = p->next_page; - } - p->next_page = NULL; - p = gc_free_pheaders = head; - } - gc_free_pheaders = p->next_page; - memset(p,0,sizeof(gc_pheader)); - p->base = (unsigned char*)base; - p->page_size = size; - -# ifdef HL_64 - void *ptr = gc_will_collide(p->base,size); - if( ptr ) { -# ifdef HL_VCC - printf("GC Page HASH collide %IX %IX\n",(int_val)GC_GET_PAGE(ptr),(int_val)ptr); -# else - printf("GC Page HASH collide %lX %lX\n",(int_val)GC_GET_PAGE(ptr),(int_val)ptr); -# endif - return gc_alloc_page(size, kind, block_count); - } -#endif - -# if defined(GC_DEBUG) - memset(base,0xDD,size); - p->page_id = PAGE_ID++; -# else - // prevent false positive to access invalid type - if( kind == MEM_KIND_DYNAMIC ) memset(base, 0, size); -# endif - if( ((int_val)base) & ((1<page_size = size; - p->page_kind = kind; - p->bmp = NULL; - - // update stats - gc_stats.pages_count++; - gc_stats.pages_allocated++; - gc_stats.pages_blocks += block_count; - gc_stats.pages_total_memory += size; - gc_stats.mark_bytes += (block_count + 7) >> 3; - - // register page in page map - int i; - for(i=0;i>GC_MASK_BITS;i++) { - void *ptr = p->base + (i<page_size>>GC_MASK_BITS;i++) { - void *ptr = ph->base + (i<page_size; - gc_stats.mark_bytes -= (block_count + 7) >> 3; - gc_free_page_memory(ph->base,ph->page_size); - ph->next_page = gc_free_pheaders; - gc_free_pheaders = ph; -} - -static void gc_check_mark(); - -void *hl_gc_alloc_gen( hl_type *t, int size, int flags ) { - void *ptr; - int time = 0; - int allocated = 0; - if( size == 0 ) - return NULL; - gc_global_lock(true); - gc_check_mark(); -# ifdef GC_MEMCHK - size += HL_WSIZE; -# endif - if( gc_flags & GC_PROFILE ) time = TIMESTAMP(); - { - allocated = size; - gc_stats.allocation_count++; - gc_stats.total_requested += size; - ptr = gc_allocator_alloc(&allocated,flags & PAGE_KIND_MASK); - if( ptr == NULL ) { - if( allocated < 0 ) { - gc_global_lock(false); - hl_error("Required memory allocation too big"); - } - hl_fatal("TODO"); - } - gc_stats.total_allocated += allocated; - } - if( gc_flags & GC_PROFILE ) gc_stats.alloc_time += TIMESTAMP() - time; -# ifdef GC_DEBUG - memset(ptr,0xCD,allocated); -# endif - if( flags & MEM_ZERO ) - MZERO(ptr,allocated); - else if( MEM_HAS_PTR(flags) && allocated != size ) - MZERO((char*)ptr+size,allocated-size); // erase possible pointers after data -# ifdef GC_MEMCHK - memset((char*)ptr+(allocated - HL_WSIZE),0xEE,HL_WSIZE); -# endif - gc_global_lock(false); - hl_track_call(HL_TRACK_ALLOC, on_alloc(t,size,flags,ptr)); - return ptr; -} - -// ------------------------- MARKING ---------------------------------------------------------- - -static float gc_mark_threshold = 0.2f; -static int mark_size = 0; -static unsigned char *mark_data = NULL; -static void **cur_mark_stack = NULL; -static void **mark_stack_end = NULL; -static int mark_stack_size = 0; - -#define GC_PUSH_GEN(ptr,page) \ - if( MEM_HAS_PTR((page)->page_kind) ) { \ - if( mark_stack == mark_stack_end ) mark_stack = hl_gc_mark_grow(mark_stack); \ - *mark_stack++ = ptr; \ - } - -HL_PRIM void **hl_gc_mark_grow( void **stack ) { - int nsize = mark_stack_size ? (((mark_stack_size * 3) >> 1) & ~1) : 256; - void **nstack = (void**)malloc(sizeof(void**) * nsize); - void **base_stack = mark_stack_end - mark_stack_size; - int avail = (int)(stack - base_stack); - if( nstack == NULL ) { - out_of_memory("markstack"); - return NULL; - } - memcpy(nstack, base_stack, avail * sizeof(void*)); - free(base_stack); - mark_stack_size = nsize; - mark_stack_end = nstack + nsize; - cur_mark_stack = nstack + avail; - if( avail == 0 ) - *cur_mark_stack++ = 0; - return cur_mark_stack; -} - -#define GC_PRECISE - -static void gc_flush_mark() { - register void **mark_stack = cur_mark_stack; - while( true ) { - void **block = (void**)*--mark_stack; - gc_pheader *page = GC_GET_PAGE(block); - unsigned int *mark_bits = NULL; - int pos = 0, nwords; -# ifdef GC_DEBUG - vdynamic *ptr = (vdynamic*)block; - ptr += 0; // prevent unreferenced warning -# endif - if( !block ) { - mark_stack++; - break; - } - int size = gc_allocator_fast_block_size(page, block); -# ifdef GC_DEBUG - if( size <= 0 ) hl_fatal("assert"); -# endif - nwords = size / HL_WSIZE; -# ifdef GC_PRECISE - if( page->page_kind == MEM_KIND_DYNAMIC ) { - hl_type *t = *(hl_type**)block; -# ifdef GC_DEBUG -# ifdef HL_64 - if( (int_val)t == 0xDDDDDDDDDDDDDDDD ) continue; -# else - if( (int_val)t == 0xDDDDDDDD ) continue; -# endif -# endif - if( t && t->mark_bits && t->kind != HFUN ) { - mark_bits = t->mark_bits; - if( t->kind == HENUM ) { - mark_bits += ((venum*)block)->index; - block += 2; - nwords -= 2; - } else { - block++; - pos++; - } - } - } -# endif - while( pos < nwords ) { - void *p; - if( mark_bits && (mark_bits[pos >> 5] & (1 << (pos&31))) == 0 ) { - pos++; - block++; - continue; - } - p = *block++; - pos++; - page = GC_GET_PAGE(p); - if( !page || !INPAGE(p,page) ) continue; - int bid = gc_allocator_get_block_id(page,p); - if( bid >= 0 && (page->bmp[bid>>3] & (1<<(bid&7))) == 0 ) { - page->bmp[bid>>3] |= 1<<(bid&7); - GC_PUSH_GEN(p,page); - } - } - } - cur_mark_stack = mark_stack; -} - -static void gc_mark_stack( void *start, void *end ) { - void **mark_stack = cur_mark_stack; - void **stack_head = (void**)start; - while( stack_head < (void**)end ) { - void *p = *stack_head++; - gc_pheader *page = GC_GET_PAGE(p); - if( !page || !INPAGE(p,page) ) continue; -# ifdef GC_INTERIOR_POINTERS - int bid = gc_allocator_get_block_interior(page, &p); -# else - int bid = gc_allocator_get_block_id(page, p); -# endif - if( bid >= 0 && (page->bmp[bid>>3] & (1<<(bid&7))) == 0 ) { - page->bmp[bid>>3] |= 1<<(bid&7); - GC_PUSH_GEN(p,page); - } - } - cur_mark_stack = mark_stack; -} - -static void gc_mark() { - void **mark_stack = cur_mark_stack; - int mark_bytes = gc_stats.mark_bytes; - int i; - // prepare mark bits - if( mark_bytes > mark_size ) { - gc_free_page_memory(mark_data, mark_size); - if( mark_size == 0 ) mark_size = GC_PAGE_SIZE; - while( mark_size < mark_bytes ) - mark_size <<= 1; - mark_data = gc_alloc_page_memory(mark_size); - if( mark_data == NULL ) out_of_memory("markbits"); - } - MZERO(mark_data,mark_bytes); - gc_allocator_before_mark(mark_data); - // push roots - for(i=0;i= 0 && (page->bmp[bid>>3] & (1<<(bid&7))) == 0 ) { - page->bmp[bid>>3] |= 1<<(bid&7); - GC_PUSH_GEN(p,page); - } - } - - // scan threads stacks & registers - for(i=0;istack_cur,t->stack_top); - gc_mark_stack(&t->gc_regs,(void**)&t->gc_regs + (sizeof(jmp_buf) / sizeof(void*) - 1)); - mark_stack = cur_mark_stack; - } - - cur_mark_stack = mark_stack; - if( mark_stack ) gc_flush_mark(); - gc_allocator_after_mark(); -} - -static void gc_major() { - int time = TIMESTAMP(), dt; - gc_stats.last_mark = gc_stats.total_allocated; - gc_stats.last_mark_allocs = gc_stats.allocation_count; - gc_stop_world(true); - gc_mark(); - gc_stop_world(false); - dt = TIMESTAMP() - time; - gc_stats.mark_count++; - gc_stats.mark_time += dt; - if( gc_flags & GC_PROFILE ) { - printf("GC-PROFILE %d\n\tmark-time %.3g\n\talloc-time %.3g\n\ttotal-mark-time %.3g\n\ttotal-alloc-time %.3g\n\tallocated %d (%dKB)\n", - gc_stats.mark_count, - dt/1000., - (gc_stats.alloc_time - last_profile.alloc_time)/1000., - gc_stats.mark_time/1000., - gc_stats.alloc_time/1000., - (int)(gc_stats.allocation_count - last_profile.allocation_count), - (int)((gc_stats.total_allocated - last_profile.total_allocated)>>10) - ); - last_profile.allocation_count = gc_stats.allocation_count; - last_profile.alloc_time = gc_stats.alloc_time; - last_profile.total_allocated = gc_stats.total_allocated; - } -} - -HL_API void hl_gc_major() { - gc_global_lock(true); - gc_major(); - gc_global_lock(false); -} - -HL_API bool hl_is_gc_ptr( void *ptr ) { - gc_pheader *page = GC_GET_PAGE(ptr); - if( !page || !INPAGE(ptr,page) ) return false; - int bid = gc_allocator_get_block_id(page, ptr); - if( bid < 0 ) return false; - //if( page->bmp && page->next_block == page->first_block && (page->bmp[bid>>3]&(1<<(bid&7))) == 0 ) return false; - return true; -} - -static bool gc_is_active = true; - -static void gc_check_mark() { - int64 m = gc_stats.total_allocated - gc_stats.last_mark; - int64 b = gc_stats.allocation_count - gc_stats.last_mark_allocs; - if( (m > gc_stats.pages_total_memory * gc_mark_threshold || b > gc_stats.pages_blocks * gc_mark_threshold || (gc_flags & GC_FORCE_MAJOR)) && gc_is_active ) - gc_major(); -} - -static void hl_gc_init() { - int i; - for(i=0;i<1<gc_blocking > 0; -} - -HL_API void hl_blocking( bool b ) { - hl_thread_info *t = current_thread; - if( !t ) - return; // allow hl_blocking in non-GC threads - if( b ) { -# ifdef HL_THREADS - if( t->gc_blocking == 0 ) - gc_save_context(t); -# endif - t->gc_blocking++; - } else if( t->gc_blocking == 0 ) - hl_error("Unblocked thread"); - else { - t->gc_blocking--; - if( t->gc_blocking == 0 && gc_threads.stopping_world ) { - gc_global_lock(true); - gc_global_lock(false); - } - } -} - -void hl_cache_free(); -void hl_cache_init(); - -void hl_global_init() { - hl_gc_init(); - hl_cache_init(); -} - -void hl_global_free() { - hl_cache_free(); -} - -struct hl_alloc_block { - int size; - hl_alloc_block *next; - unsigned char *p; -}; - -void hl_alloc_init( hl_alloc *a ) { - a->cur = NULL; -} - -void *hl_malloc( hl_alloc *a, int size ) { - hl_alloc_block *b = a->cur; - void *p; - if( !size ) return NULL; - size += hl_pad_size(size,&hlt_dyn); - if( b == NULL || b->size <= size ) { - int alloc = size < 4096-sizeof(hl_alloc_block) ? 4096-sizeof(hl_alloc_block) : size; - b = (hl_alloc_block *)malloc(sizeof(hl_alloc_block) + alloc); - if( b == NULL ) out_of_memory("malloc"); - b->p = ((unsigned char*)b) + sizeof(hl_alloc_block); - b->size = alloc; - b->next = a->cur; - a->cur = b; - } - p = b->p; - b->p += size; - b->size -= size; - return p; -} - -void *hl_zalloc( hl_alloc *a, int size ) { - void *p = hl_malloc(a,size); - if( p ) MZERO(p,size); - return p; -} - -void hl_free( hl_alloc *a ) { - hl_alloc_block *b = a->cur; - int_val prev = 0; - int size = 0; - while( b ) { - hl_alloc_block *n = b->next; - size = (int)(b->p + b->size - ((unsigned char*)b)); - prev = (int_val)b; - free(b); - b = n; - } - // check if our allocator was not part of the last free block - if( (int_val)a < prev || (int_val)a > prev+size ) - a->cur = NULL; -} - -HL_PRIM void *hl_alloc_executable_memory( int size ) { -#ifdef __APPLE__ -# ifndef MAP_ANONYMOUS -# define MAP_ANONYMOUS MAP_ANON -# endif -#endif -#if defined(HL_WIN) && defined(HL_64) - static char *jit_address = (char*)0x000076CA9F000000; - void *ptr; -retry_jit_alloc: - ptr = VirtualAlloc(jit_address,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE); - if( !ptr ) { - jit_address = (char*)(((int_val)jit_address)>>1); // fix for Win7 - will eventually reach NULL - goto retry_jit_alloc; - } - jit_address += size + ((-size) & (GC_PAGE_SIZE - 1)); - return ptr; -#elif defined(HL_WIN) - void *ptr = VirtualAlloc(NULL,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE); - return ptr; -#elif defined(HL_CONSOLE) - return NULL; -#else - void *p; - p = mmap(NULL,size,PROT_READ|PROT_WRITE|PROT_EXEC,(MAP_PRIVATE|MAP_ANONYMOUS),-1,0); - return p; -#endif -} - -HL_PRIM void hl_free_executable_memory( void *c, int size ) { -#if defined(HL_WIN) - VirtualFree(c,0,MEM_RELEASE); -#elif !defined(HL_CONSOLE) - munmap(c, size); -#endif -} - -#if defined(HL_CONSOLE) -void *sys_alloc_align( int size, int align ); -void sys_free_align( void *ptr, int size ); -#elif !defined(HL_WIN) -static void *base_addr = (void*)0x40000000; -#endif - -static void *gc_alloc_page_memory( int size ) { -#if defined(HL_WIN) -# if defined(GC_DEBUG) && defined(HL_64) -# define STATIC_ADDRESS -# endif -# ifdef STATIC_ADDRESS - // force out of 32 bits addresses to check loss of precision - static char *start_address = (char*)0x100000000; -# else - static void *start_address = NULL; -# endif - void *ptr = VirtualAlloc(start_address,size,MEM_RESERVE|MEM_COMMIT,PAGE_READWRITE); -# ifdef STATIC_ADDRESS - if( ptr == NULL && start_address ) { - start_address = NULL; - return gc_alloc_page_memory(size); - } - start_address += size + ((-size) & (GC_PAGE_SIZE - 1)); -# endif - return ptr; -#elif defined(HL_CONSOLE) - return sys_alloc_align(size, GC_PAGE_SIZE); -#else - int i = 0; - while( gc_will_collide(base_addr,size) ) { - base_addr = (char*)base_addr + GC_PAGE_SIZE; - i++; - // most likely our hashing creates too many collisions - if( i >= 1 << (GC_LEVEL0_BITS + GC_LEVEL1_BITS + 2) ) - return NULL; - } - void *ptr = mmap(base_addr,size,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); - if( ptr == (void*)-1 ) - return NULL; - if( ((int_val)ptr) & (GC_PAGE_SIZE-1) ) { - munmap(ptr,size); - void *tmp; - int tmp_size = (int)((int_val)ptr - (int_val)base_addr); - if( tmp_size > 0 ) { - base_addr = (void*)((((int_val)ptr) & ~(GC_PAGE_SIZE-1)) + GC_PAGE_SIZE); - tmp = ptr; - } else { - base_addr = (void*)(((int_val)ptr) & ~(GC_PAGE_SIZE-1)); - tmp = NULL; - } - if( tmp ) tmp = mmap(tmp,tmp_size,PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); - ptr = gc_alloc_page_memory(size); - if( tmp ) munmap(tmp,tmp_size); - return ptr; - } - base_addr = (char*)ptr+size; - return ptr; -#endif -} - -static void gc_free_page_memory( void *ptr, int size ) { -#ifdef HL_WIN - VirtualFree(ptr, 0, MEM_RELEASE); -#elif defined(HL_CONSOLE) - sys_free_align(ptr,size); -#else - munmap(ptr,size); -#endif -} - -vdynamic *hl_alloc_dynamic( hl_type *t ) { - vdynamic *d = (vdynamic*)hl_gc_alloc_gen(t, sizeof(vdynamic), (hl_is_ptr(t) ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR) | MEM_ZERO); - d->t = t; - return d; -} - -#ifndef HL_64 -# define DYN_PAD 0, -#else -# define DYN_PAD -#endif - -static const vdynamic vdyn_true = { &hlt_bool, DYN_PAD {true} }; -static const vdynamic vdyn_false = { &hlt_bool, DYN_PAD {false} }; - -vdynamic *hl_alloc_dynbool( bool b ) { - return (vdynamic*)(b ? &vdyn_true : &vdyn_false); -} - - -vdynamic *hl_alloc_obj( hl_type *t ) { - vobj *o; - int size; - int i; - hl_runtime_obj *rt = t->obj->rt; - if( rt == NULL || rt->methods == NULL ) rt = hl_get_obj_proto(t); - size = rt->size; - if( size & (HL_WSIZE-1) ) size += HL_WSIZE - (size & (HL_WSIZE-1)); - if( t->kind == HSTRUCT ) { - o = (vobj*)hl_gc_alloc_gen(t, size, (rt->hasPtr ? MEM_KIND_RAW : MEM_KIND_NOPTR) | MEM_ZERO); - } else { - o = (vobj*)hl_gc_alloc_gen(t, size, (rt->hasPtr ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR) | MEM_ZERO); - o->t = t; - } - for(i=0;inbindings;i++) { - hl_runtime_binding *b = rt->bindings + i; - *(void**)(((char*)o) + rt->fields_indexes[b->fid]) = b->closure ? hl_alloc_closure_ptr(b->closure,b->ptr,o) : b->ptr; - } - return (vdynamic*)o; -} - -vdynobj *hl_alloc_dynobj() { - vdynobj *o = (vdynobj*)hl_gc_alloc_gen(&hlt_dynobj,sizeof(vdynobj),MEM_KIND_DYNAMIC | MEM_ZERO); - o->t = &hlt_dynobj; - return o; -} - -vvirtual *hl_alloc_virtual( hl_type *t ) { - vvirtual *v = (vvirtual*)hl_gc_alloc(t, t->virt->dataSize + sizeof(vvirtual) + sizeof(void*) * t->virt->nfields); - void **fields = (void**)(v + 1); - char *vdata = (char*)(fields + t->virt->nfields); - int i; - v->t = t; - v->value = NULL; - v->next = NULL; - for(i=0;ivirt->nfields;i++) - fields[i] = (char*)v + t->virt->indexes[i]; - MZERO(vdata,t->virt->dataSize); - return v; -} - -HL_API void hl_gc_stats( double *total_allocated, double *allocation_count, double *current_memory ) { - *total_allocated = (double)gc_stats.total_allocated; - *allocation_count = (double)gc_stats.allocation_count; - *current_memory = (double)gc_stats.pages_total_memory; -} - -HL_API void hl_gc_enable( bool b ) { - gc_is_active = b; -} - -HL_API int hl_gc_get_flags() { - return gc_flags; -} - -HL_API void hl_gc_set_flags( int f ) { - gc_flags = f; -} - -HL_API void hl_gc_profile( bool b ) { - if( b ) - gc_flags |= GC_PROFILE; - else - gc_flags &= GC_PROFILE; -} - -static FILE *fdump; -static void fdump_i( int i ) { - fwrite(&i,1,4,fdump); -} -static void fdump_p( void *p ) { - fwrite(&p,1,sizeof(void*),fdump); -} -static void fdump_d( void *p, int size ) { - fwrite(p,1,size,fdump); -} - -static hl_types_dump gc_types_dump = NULL; -HL_API void hl_gc_set_dump_types( hl_types_dump tdump ) { - gc_types_dump = tdump; -} - -HL_API void hl_gc_dump_memory( const char *filename ) { - int i; - gc_global_lock(true); - gc_stop_world(true); - gc_mark(); - fdump = fopen(filename,"wb"); - // header - fdump_d("HMD0",4); - fdump_i(((sizeof(void*) == 8)?1:0) | ((sizeof(bool) == 4)?2:0)); - // pages -/* - fdump_i(GC_ALL_PAGES); - for(i=0;ibase); - fdump_i(p->page_kind); - fdump_i(p->page_size); - fdump_i(p->block_size); - fdump_i(p->first_block); - fdump_i(p->max_blocks); - fdump_i(p->next_block); - fdump_d(p->base,p->page_size); - fdump_i((p->bmp ? 1 :0) | (p->sizes?2:0)); - if( p->bmp ) fdump_d(p->bmp,(p->max_blocks + 7) >> 3); - if( p->sizes ) fdump_d(p->sizes,p->max_blocks); - p = p->next_page; - } - fdump_p(NULL); - } -*/ - // roots - fdump_i(gc_roots_count); - for(i=0;istack_top); - int size = (int)((void**)t->stack_top - (void**)t->stack_cur); - fdump_i(size); - fdump_d(t->stack_cur,size*sizeof(void*)); - } - // types -# define fdump_t(t) fdump_i(t.kind); fdump_p(&t); - fdump_t(hlt_i32); - fdump_t(hlt_f32); - fdump_t(hlt_f64); - fdump_t(hlt_dyn); - fdump_t(hlt_array); - fdump_t(hlt_bytes); - fdump_t(hlt_dynobj); - fdump_t(hlt_bool); - fdump_i(-1); - if( gc_types_dump ) gc_types_dump(fdump_d); - fclose(fdump); - fdump = NULL; - gc_stop_world(false); - gc_global_lock(false); -} - -#ifdef HL_VCC -# pragma optimize( "", off ) -#endif -HL_API vdynamic *hl_debug_call( int mode, vdynamic *v ) { - return NULL; -} -#ifdef HL_VCC -# pragma optimize( "", on ) -#endif - -DEFINE_PRIM(_VOID, gc_major, _NO_ARG); -DEFINE_PRIM(_VOID, gc_enable, _BOOL); -DEFINE_PRIM(_VOID, gc_profile, _BOOL); -DEFINE_PRIM(_VOID, gc_stats, _REF(_F64) _REF(_F64) _REF(_F64)); -DEFINE_PRIM(_VOID, gc_dump_memory, _BYTES); -DEFINE_PRIM(_I32, gc_get_flags, _NO_ARG); -DEFINE_PRIM(_VOID, gc_set_flags, _I32); -DEFINE_PRIM(_DYN, debug_call, _I32 _DYN); -DEFINE_PRIM(_VOID, blocking, _BOOL); +#include "hl.h" +#include "gc.h" + +#ifdef HL_WIN +# include +#else +# include +# include +#endif + + +#include +#include +#include + +#ifdef __APPLE__ +# ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +# endif +#endif + +#ifdef HL_THREADS +#error "no threads with gc for now" +#endif + +#define GC_STATIC + +// utilities -------------------------------------------------------- + +// branch optimisation +#define LIKELY(c) __builtin_expect((c), 1) +#define UNLIKELY(c) __builtin_expect((c), 0) + +// bit access +#define GET_BIT(bmp, bit) ((bmp)[(bit) >> 3] & (1 << ((bit) & 7))) +#define SET_BIT0(bmp, bit) (bmp)[(bit) >> 3] &= 0xFF ^ (1 << ((bit) & 7)); +#define SET_BIT1(bmp, bit) (bmp)[(bit) >> 3] |= 1 << ((bit) & 7); +#define SET_BIT(bmp, bit, val) (bmp)[(bit) >> 3] = ((bmp)[(bit) >> 3] & (0xFF ^ (1 << ((bit) & 7)))) | ((val) << ((bit) & 7)); + +// doubly-linked lists +#define DLL_UNLINK(obj) do { \ + (obj)->next->prev = (obj)->prev; \ + (obj)->prev->next = (obj)->next; \ + (obj)->next = NULL; \ + (obj)->prev = NULL; \ + } while (0) +#define DLL_INSERT(obj, head) do { \ + (obj)->next = (head)->next; \ + (obj)->prev = (head); \ + (head)->next = (obj); \ + (obj)->next->prev = (obj); \ + } while (0) + +// OS page management ---------------------------------------------- + +static void *base_addr; + +// allocates a page-aligned region of memory from the OS +GC_STATIC void *gc_alloc_os_memory(int size) { + GC_DEBUG_DUMP1("gc_alloc_os_memory.enter", size); + if (gc_stats.total_memory + size >= gc_config.memory_limit) { + GC_DEBUG_DUMP0("gc_alloc_os_memory.fail.oom"); + GC_DEBUG(fatal, "using %lu / %lu bytes, need %d more", gc_stats.total_memory, gc_config.memory_limit, size); + GC_FATAL("OOM: memory limit hit"); + } + void *ptr = mmap(base_addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == (void *)-1) { + GC_DEBUG_DUMP0("gc_alloc_os_memory.fail.other"); + GC_FATAL("failed to allocate page"); + } + gc_stats.total_memory += size; + if (((int_val)ptr) & (GC_PAGE_SIZE - 1)) { + // re-align + munmap(ptr,size); + void *tmp; + int tmp_size = (int)((int_val)ptr - (int_val)base_addr); + if (tmp_size > 0) { + base_addr = (void *)((((int_val)ptr) & ~(GC_PAGE_SIZE - 1)) + GC_PAGE_SIZE); + tmp = ptr; + } else { + base_addr = (void *)(((int_val)ptr) & ~(GC_PAGE_SIZE - 1)); + tmp = NULL; + } + if (tmp) + tmp = mmap(tmp, tmp_size, PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + // TODO: too many retries ??? + ptr = gc_alloc_os_memory(size); + if (tmp) + munmap(tmp, tmp_size); + return ptr; + } + base_addr = (char *)ptr + size; + GC_DEBUG_DUMP2("gc_alloc_os_memory.success", ptr, size); + GC_DEBUG(os, "allocated OS page: %p", ptr); + return ptr; +} + +// returns a region of memory back to the OS +GC_STATIC void gc_free_os_memory(void *ptr, int size) { + munmap(ptr, size); + gc_stats.total_memory -= size; + GC_DEBUG_DUMP2("gc_free_os_memory.success", ptr, size); + GC_DEBUG(os, "reclaimed OS page: %p", ptr); +} + +// GC page management ---------------------------------------------- + +static void *gc_min_allocated; +static void *gc_max_allocated; +static gc_page_header_t *gc_last_allocated_page; +static gc_page_header_t *gc_last_allocated_huge_page; +static gc_page_hash_t gc_page_hash; +static hl_mutex *gc_mutex_pool; +static gc_block_header_t *gc_pool; +static int gc_pool_count; +// TODO: pre-allocate page (or pages) to be used for fast allocation (refresh during collection) +// TODO: mutexes ... + +// allocates and initialises a GC page +GC_STATIC void gc_add_page(gc_page_header_t *header) { + if ((char *)header < (char *)gc_min_allocated) { + gc_min_allocated = (char *)header; + } + if ((char *)header + GC_PAGE_SIZE > (char *)gc_max_allocated) { + gc_max_allocated = (char *)header + GC_PAGE_SIZE; + } + + gc_stats.total_pages++; + gc_page_hash.total_pages++; + double load_factor = gc_page_hash.total_pages / gc_page_hash.bucket_count; + if (load_factor > 0.75) { // TODO: limit the growth of buckets + gc_page_hash.bucket_count <<= 1; + free(gc_page_hash.buckets); + gc_page_hash.buckets = calloc(sizeof(gc_page_header_t *), gc_page_hash.bucket_count); + for (gc_page_header_t *cur = gc_last_allocated_page; cur != NULL; cur = cur->next_page) { + int bucket = GC_HASH(cur) % gc_page_hash.bucket_count; + cur->next_page_bucket = gc_page_hash.buckets[bucket]; + gc_page_hash.buckets[bucket] = cur; + } + for (gc_page_header_t *cur = gc_last_allocated_huge_page; cur != NULL; cur = cur->next_page) { + int bucket = GC_HASH(cur) % gc_page_hash.bucket_count; + cur->next_page_bucket = gc_page_hash.buckets[bucket]; + gc_page_hash.buckets[bucket] = cur; + } + } + int bucket = GC_HASH(header) % gc_page_hash.bucket_count; + header->next_page_bucket = gc_page_hash.buckets[bucket]; + gc_page_hash.buckets[bucket] = header; + GC_DEBUG(page, "page %p in bucket %d", header, bucket); +} + +// allocates a normal page (containing immix blocks) +GC_STATIC gc_page_header_t *gc_alloc_page_normal(void) { + gc_page_header_t *header = (gc_page_header_t *)gc_alloc_os_memory(GC_PAGE_SIZE); + if (header == NULL) { + // TODO: OOM here rather than in gc_pop_block ? + return NULL; + } + + header->size = GC_PAGE_SIZE; + header->free_blocks = GC_BLOCKS_PER_PAGE; + header->kind = GC_PAGE_NORMAL; + header->next_page = gc_last_allocated_page; + gc_last_allocated_page = header; + + gc_add_page(header); + + gc_block_header_t *blocks = GC_PAGE_BLOCK(header, 0); + for (int i = 0; i < GC_BLOCKS_PER_PAGE; i++) { + blocks[i].kind = GC_BLOCK_FREE; + blocks[i].next = &blocks[i + 1]; + } + hl_mutex_acquire(gc_mutex_pool); + blocks[GC_BLOCKS_PER_PAGE - 1].next = gc_pool; + gc_pool = &blocks[0]; + gc_pool_count += GC_BLOCKS_PER_PAGE; + hl_mutex_release(gc_mutex_pool); + + GC_DEBUG_DUMP1("gc_alloc_page_normal.success", header); + return header; +} + +// allocates a huge page (containing a single object) +GC_STATIC gc_page_header_t *gc_alloc_page_huge(int size) { + gc_page_header_t *header = (gc_page_header_t *)gc_alloc_os_memory(sizeof(gc_page_header_t) + size); + if (header == NULL) { + GC_FATAL("OOM: huge page"); + } + + header->size = sizeof(gc_page_header_t) + size; + header->kind = GC_PAGE_HUGE; + header->next_page = gc_last_allocated_huge_page; + gc_last_allocated_huge_page = header; + + gc_add_page(header); + + GC_DEBUG_DUMP2("gc_alloc_page_huge.success", header, size); + return header; +} + +// frees a GC page +GC_STATIC void gc_free_page_memory(gc_page_header_t *header) { + // TODO: make page list a DLL + gc_page_header_t **last = header->kind == GC_PAGE_HUGE ? &gc_last_allocated_huge_page : &gc_last_allocated_page; + for (gc_page_header_t *cur = *last; cur != NULL; cur = cur->next_page) { + if (cur == header) { + *last = cur->next_page; + break; + } + last = &cur->next_page; + } + int bucket = GC_HASH(header) % gc_page_hash.bucket_count; + last = &gc_page_hash.buckets[bucket]; + for (gc_page_header_t *cur = gc_page_hash.buckets[bucket]; cur != NULL; cur = cur->next_page_bucket) { + if (cur == header) { + *last = cur->next_page_bucket; + break; + } + last = &cur->next_page; + } + // TODO: maybe rearrange if load factor is low? + gc_page_hash.total_pages--; + gc_stats.total_pages--; + gc_free_os_memory(header, header->size); + GC_DEBUG_DUMP1("gc_free_page_memory.success", header); +} + +// roots ----------------------------------------------------------- + +static hl_mutex *gc_mutex_roots; +static void ***gc_roots; +static int gc_root_count; +static int gc_root_capacity; + +// adds a root to the GC root list +// a root is a mutable pointer to GC-allocated memory (or NULL), which is +// dereferenced and followed when starting the GC marking phase +HL_PRIM void hl_add_root(void **p) { + if (p == NULL) { + return; + } + hl_mutex_acquire(gc_mutex_roots); + if (gc_root_count >= gc_root_capacity) { + int new_capacity = gc_root_capacity << 1; + if (gc_root_capacity == 0) { + new_capacity = 128; + } + void ***new_roots = (void ***)calloc(new_capacity, sizeof(void **)); + if (gc_roots != NULL) { + memcpy(new_roots, gc_roots, sizeof(void **) * gc_root_capacity); + } + gc_roots = new_roots; + gc_root_capacity = new_capacity; + } + gc_roots[gc_root_count++] = p; + hl_mutex_release(gc_mutex_roots); + GC_DEBUG_DUMP1("hl_add_root.success", p); +} + +// removes the given root from the GC root list +HL_PRIM void hl_remove_root(void **p) { + hl_mutex_acquire(gc_mutex_roots); + for (int i = 0; i < gc_root_count; i++) { + if (gc_roots[i] == p) { + gc_roots[i] = gc_roots[--gc_root_count]; + break; + } + } + hl_mutex_release(gc_mutex_roots); + GC_DEBUG_DUMP1("hl_remove_root.success", p); +} + +// thread setup ---------------------------------------------------- + +HL_THREAD_STATIC_VAR hl_thread_info *current_thread; +static int gc_mark_polarity; +static int gc_mark_total; +static gc_mark_stack_t gc_mark_stack; +static gc_mark_stack_t gc_mark_stack_next; +static gc_mark_stack_t *gc_mark_stack_active; + +// registers a thread with the GC +// a registered thread can use GC allocation and must be stopped when +// collection is happening +HL_API void hl_register_thread(void *stack_top) { + if (hl_get_thread()) + hl_fatal("thread already registered"); + + hl_thread_info *t = (hl_thread_info*)calloc(sizeof(hl_thread_info), 1); + t->thread_id = hl_thread_id(); + t->stack_top = stack_top; + t->flags = HL_TRACK_MASK << HL_TREAD_TRACK_SHIFT; + + gc_block_header_t *block = gc_pop_block(); + t->lines_block = block; + t->lines_start = &block->lines[0]; + t->lines_limit = &block->lines[GC_LINES_PER_BLOCK]; + + gc_block_dummy_t *sentinels = (gc_block_dummy_t *)calloc(sizeof(gc_block_dummy_t), 4); + sentinels[0].next = (gc_block_header_t *)&sentinels[1]; + sentinels[1].prev = (gc_block_header_t *)&sentinels[0]; + sentinels[2].next = (gc_block_header_t *)&sentinels[3]; + sentinels[3].prev = (gc_block_header_t *)&sentinels[2]; + t->sentinels = sentinels; + + t->full_blocks = (gc_block_header_t *)&sentinels[0]; + t->recyclable_blocks = (gc_block_header_t *)&sentinels[2]; + + current_thread = t; + hl_add_root((void **)&t->exc_value); + hl_add_root((void **)&t->exc_handler); + + /* + // TODO: thread list + gc_global_lock(true); + hl_thread_info **all = (hl_thread_info**)malloc(sizeof(void*) * (gc_threads.count + 1)); + memcpy(all,gc_threads.threads,sizeof(void*)*gc_threads.count); + gc_threads.threads = all; + all[gc_threads.count++] = t; + gc_global_lock(false); + */ +} + +// unregisters a thread from the GC +HL_API void hl_unregister_thread(void) { + hl_thread_info *t = hl_get_thread(); + if (!t) + hl_fatal("thread not registered"); + hl_remove_root((void **)&t->exc_value); + hl_remove_root((void **)&t->exc_handler); + + // TODO: release blocks into zombie stack + free(t->sentinels); + free(t); + current_thread = NULL; + + /* + gc_global_lock(true); + for(int i = 0;i < gc_threads.count; i++) + if( gc_threads.threads[i] == t ) { + memmove(gc_threads.threads + i, gc_threads.threads + i + 1, sizeof(void*) * (gc_threads.count - i - 1)); + gc_threads.count--; + break; + } + free(t); + current_thread = NULL; + // don't use gc_global_lock(false) + hl_mutex_release(gc_threads.global_lock); + */ +} + +HL_API void *hl_gc_threads_info() { + return NULL; +} + +GC_STATIC void gc_save_context(hl_thread_info *t) { + setjmp(t->gc_regs); + t->stack_cur = &t; +} + +GC_STATIC void gc_stop_world( bool b ) { +# ifdef HL_THREADS + if( b ) { + int i; + gc_threads.stopping_world = true; + for(i=0;igc_blocking == 0 ) {}; // spinwait + } + } else { + // releasing global lock will release all threads + gc_threads.stopping_world = false; + } +# else + if (b) { + gc_save_context(current_thread); + } +# endif +} + +// thread-local allocator <-> global GC ---------------------------- + +// acquires a block for the current thread +// the block may come from the thread's recycle list, or from the global +// free block pool (which incurs a locking penalty) +GC_STATIC gc_block_header_t *gc_pop_block(void) { + hl_mutex_acquire(gc_mutex_pool); // TODO: mutexes need to be re-entrant, or lock later + if (UNLIKELY(gc_pool_count == 0)) { + // if (gc_stats.total_memory * 1.2 >= gc_config.memory_limit) { + if (gc_stats.live_blocks > 0) { + GC_DEBUG(alloc, "using %lu / %lu bytes", gc_stats.total_memory, gc_config.memory_limit); + // GC_FATAL("OOM: memory limit hit"); + GC_DEBUG(alloc, "memory limit hit, triggering major ..."); + hl_gc_major(); + } + if (gc_pool_count == 0 && gc_alloc_page_normal() == NULL) { + GC_FATAL("OOM: page memory"); + } + } + gc_block_header_t *block = gc_pool; + if ((((int_val)block->next) & (int_val)(0x10000 - 1)) != 0) { + GC_FATAL("pool corrupted in pop"); + } + gc_pool = block->next; + gc_page_header_t *page = GC_BLOCK_PAGE(block); + // TODO: mutex per page? + page->free_blocks--; + SET_BIT1(page->block_bmp, GC_BLOCK_ID(block)); + gc_pool_count--; + hl_mutex_release(gc_mutex_pool); + block->kind = GC_BLOCK_NEW; + block->owner_thread = hl_thread_id(); + GC_DEBUG(alloc, "got block %p", block); + GC_DEBUG_DUMP1("gc_pop_block.success", block); + gc_stats.live_blocks++; + return block; +} + +// returns a thread-owned block to the global free pool +GC_STATIC void gc_push_block(gc_block_header_t *block) { + if (block->kind == GC_BLOCK_FULL || block->kind == GC_BLOCK_RECYCLED) { + DLL_UNLINK(block); + } // TODO: condition should always be true, until zombie blocks are introduced ? + block->kind = GC_BLOCK_FREE; // TODO: zombie ? + block->owner_thread = -1; + memset(block->line_marks, 0, GC_LINES_PER_BLOCK); + gc_page_header_t *page = GC_BLOCK_PAGE(block); + // TODO: mutex per page? + hl_mutex_acquire(gc_mutex_pool); + page->free_blocks++; + SET_BIT0(page->block_bmp, GC_BLOCK_ID(block)); + // TODO: manage disorder in GC pool (prefer address order) + block->next = gc_pool; + if ((((int_val)block) & (int_val)(0x10000 - 1)) != 0) { + GC_FATAL("pool corrupted in push"); + } + gc_pool = block; + gc_pool_count++; + gc_stats.live_blocks--; + hl_mutex_release(gc_mutex_pool); + GC_DEBUG_DUMP1("gc_push_block.success", block); +} + +// acquires a huge object +// this always allocates an OS page +GC_STATIC void *gc_pop_huge(int size) { + gc_page_header_t *header = gc_alloc_page_huge(8192 + size); + GC_DEBUG(alloc, "allocated huge %d at %p", size, header); + GC_DEBUG_DUMP2("gc_pop_huge.success", header, size); + gc_block_header_t *block = (gc_block_header_t *)header; + block->huge_sized = 1; + return &block->lines[0]; +} + +// frees a huge object +GC_STATIC void gc_push_huge(void *huge) { + gc_page_header_t *header = GC_BLOCK_PAGE(huge); + GC_DEBUG(alloc, "freed huge %p", huge); + GC_DEBUG_DUMP1("gc_push_huge.success", header); + gc_free_os_memory(header, header->size); +} + +// thread <-> thread-local allocator ------------------------------- + +// bump allocate an object with the current bump cursor +// should only be called when there is definitely enough space to allocate +// the object +// this also initialises the object header +GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) { + gc_object_t *ret = (gc_object_t *)current_thread->lines_start; + gc_metadata_t *meta = GC_METADATA(ret); + + // clear stale data + memset(ret, 0, sizeof(void *) * words); + + ret->t = t; + meta->flags = 0; + meta->marked = !gc_mark_polarity; + meta->words = words & 15; + if (flags & GC_ALLOC_DYNAMIC) { + meta->dynamic_mark = 1; + } + if (flags & GC_ALLOC_NOPTR) { + meta->no_ptr = 1; + } + // int line_id = GC_LINE_ID(ret); + // gc_block_header_t *block = GC_LINE_BLOCK(ret); + // GC_DEBUG("pre-allocation: %p - %p", current_thread->lines_start, current_thread->lines_limit); + current_thread->lines_start = (void *)(((char *)current_thread->lines_start) + size); + if (LIKELY(size <= GC_LINE_SIZE)) { + // GC_DEBUG("allocated small %p in block %p (%d bytes, %d words)", ret, block, size, words); + } else { + meta->medium_sized = 1; + gc_metadata_ext_t *meta_ext = (gc_metadata_ext_t *)meta; + meta_ext->words_ext = words >> 4; + // int last_id = GC_LINE_ID((void *)ret + size); + // ret->line_span = (last_id - line_id) + 1; + // GC_DEBUG("allocated medium %p in block %p (%d bytes, %d lines, %d words)", ret, block, size, ret->line_span, words); + } + + GC_DEBUG_DUMP3("gc_alloc_bump.success", ret, size, flags); + return ret; +} + +// finds the next line gap in the active block of the current thread +// returns true if a gap was found +GC_STATIC bool gc_find_gap(void) { + if (GC_LINE_BLOCK(current_thread->lines_start) != current_thread->lines_block) + return false; + // TODO: optimise + int line = GC_LINE_ID(current_thread->lines_start); + bool last_free = (line == 0); + for (; line < GC_LINES_PER_BLOCK; line++) { + if (current_thread->lines_block->line_marks[line] == 0) { + // skip line gap immediately after a marked line + if (last_free) { + break; + } else { + last_free = true; + } + } else { + last_free = false; + } + } + if (line >= GC_LINES_PER_BLOCK) + return false; + current_thread->lines_start = ¤t_thread->lines_block->lines[line]; + // GC_DEBUG("gap search in %p; line %d, %p", current_thread->lines_block, line, current_thread->lines_start); + // found a gap, increase limit as far as possible + // TODO: optimise + while (current_thread->lines_block->line_marks[line] == 0) { + line++; + } + current_thread->lines_limit = ¤t_thread->lines_block->lines[line]; + return true; +} + +// generic allocation function +HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { + // align to words + int words = (size + (sizeof(void *) - 1)) / sizeof(void *); + size = words * sizeof(void *); + + if (LIKELY((char *)current_thread->lines_start + size <= (char *)current_thread->lines_limit)) { + return gc_alloc_bump(t, size, words, flags); + } + + // TODO: list operations should not be interrupted by the GC, mutex per list + + if (LIKELY(size <= GC_LINE_SIZE)) { + // find next line gap (based on mark bits from previous collection) in current block + if (gc_find_gap()) { + return gc_alloc_bump(t, size, words, flags); + } + + // add current block to full blocks + current_thread->lines_block->kind = GC_BLOCK_FULL; + DLL_INSERT(current_thread->lines_block, current_thread->full_blocks); + + // no gap found, get next recyclable block + if (current_thread->recyclable_blocks->next->next != NULL) { + current_thread->lines_block = current_thread->recyclable_blocks->next; + current_thread->lines_start = ¤t_thread->lines_block->lines[0]; + DLL_UNLINK(current_thread->lines_block); + current_thread->lines_block->kind = GC_BLOCK_NEW; // TODO: maybe a separate kind + if (gc_find_gap()) { + return gc_alloc_bump(t, size, words, flags); + } + GC_FATAL("unreachable"); + } + + // if none, get fresh block + current_thread->lines_block = gc_pop_block(); + if (current_thread->lines_block == NULL) { + hl_gc_major(); + current_thread->lines_block = gc_pop_block(); + if (current_thread->lines_block == NULL) { + GC_FATAL("OOM: alloc"); + } + } + current_thread->lines_start = ¤t_thread->lines_block->lines[0]; + current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; + return gc_alloc_bump(t, size, words, flags); + } else if (size <= GC_MEDIUM_SIZE) { + current_thread->lines_block->kind = GC_BLOCK_FULL; + DLL_INSERT(current_thread->lines_block, current_thread->full_blocks); + + // get empty block to avoid expensive search + current_thread->lines_block = gc_pop_block(); + if (current_thread->lines_block == NULL) { + hl_gc_major(); + current_thread->lines_block = gc_pop_block(); + if (current_thread->lines_block == NULL) { + GC_FATAL("OOM: alloc"); + } + } + current_thread->lines_start = ¤t_thread->lines_block->lines[0]; + current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; + return gc_alloc_bump(t, size, words, flags); + } else { + gc_object_t *obj = gc_pop_huge(size); + obj->t = t; + gc_metadata_t *meta = GC_METADATA(obj); + meta->flags = 0; + meta->marked = !gc_mark_polarity; + if (flags & GC_ALLOC_DYNAMIC) { + meta->dynamic_mark = 1; + } + if (flags & GC_ALLOC_NOPTR) { + meta->no_ptr = 1; + } + return obj; + } +} + +// triggers a major GC collection +HL_API void hl_gc_major(void) { + GC_DEBUG_DUMP0("hl_gc_major.entry"); + gc_stop_world(true); + gc_mark(); + gc_sweep(); + gc_stop_world(false); + GC_DEBUG_DUMP0("hl_gc_major.success"); +} + +// GC internals ---------------------------------------------------- + +// push a pointer (and its incoming reference) to the mark stack +GC_STATIC void gc_push(void *p, void *ref) { + if (p == NULL) + return; + if (UNLIKELY(gc_mark_stack_active->pos >= gc_mark_stack_active->capacity)) { + if (gc_mark_stack_active == &gc_mark_stack) { + GC_DEBUG(mark, "mark stack growth alloc"); + // ran out of space on original stack, allocate a new one and start using it + gc_mark_stack_next.capacity = gc_mark_stack.capacity << 1; + gc_mark_stack_next.data = malloc(sizeof(gc_mark_stack_entry_t) * gc_mark_stack_next.capacity); + gc_mark_stack_active = &gc_mark_stack_next; + } else { + GC_DEBUG(mark, "mark stack panic realloc"); + // ran out of space on both stacks, allocate a new one and copy into it + int new_capacity = gc_mark_stack_active->capacity << 1; // TODO: OOM if too large + gc_mark_stack_entry_t *new_stack = (gc_mark_stack_entry_t *)malloc(sizeof(gc_mark_stack_entry_t) * new_capacity); + memcpy(new_stack, gc_mark_stack.data, sizeof(gc_mark_stack_entry_t) * gc_mark_stack.pos); + memcpy(&new_stack[gc_mark_stack.pos], gc_mark_stack_next.data, sizeof(gc_mark_stack_entry_t) * gc_mark_stack_next.pos); + free(gc_mark_stack.data); + free(gc_mark_stack_next.data); + gc_mark_stack.data = new_stack; + gc_mark_stack_next.data = NULL; + gc_mark_stack.pos = gc_mark_stack.pos + gc_mark_stack_next.pos; + gc_mark_stack.capacity = new_capacity; + gc_mark_stack_active = &gc_mark_stack; + } + } + if (gc_get_block(p) != NULL) { + gc_object_t *obj = p; + gc_metadata_t *meta = GC_METADATA(obj); + if (meta->marked != gc_mark_polarity) { + meta->marked = gc_mark_polarity; + gc_mark_total++; + gc_mark_stack_active->data[gc_mark_stack_active->pos].object = p; + gc_mark_stack_active->data[gc_mark_stack_active->pos++].reference = ref; + GC_DEBUG(mark, "pushed %p -> %p to %p", ref, p, gc_mark_stack_active->data); + GC_DEBUG_DUMP2("gc_push.success", p, ref); + } + } +} + +// get the next pointer (and its incoming reference) from the mark stack +GC_STATIC gc_object_t *gc_pop(void) { + void *p = gc_mark_stack.data[--gc_mark_stack.pos].object; + void *ref = gc_mark_stack.data[gc_mark_stack.pos].reference; + // GC_DEBUG("[M] popped %p from %p", p, gc_mark_stack.data); + if (gc_mark_stack.pos <= 0 && gc_mark_stack_active != &gc_mark_stack) { + GC_DEBUG(mark, "mark stack shrunk"); + // emptied original stack, switch + free(gc_mark_stack.data); + gc_mark_stack.data = gc_mark_stack_next.data; + gc_mark_stack.pos = gc_mark_stack_next.pos; + gc_mark_stack.capacity = gc_mark_stack_next.capacity; + gc_mark_stack_next.data = NULL; + gc_mark_stack_active = &gc_mark_stack; + } + GC_DEBUG_DUMP2("gc_pop.success", p, ref); + return p; +} + +HL_PRIM vbyte* hl_type_name( hl_type *t ); + +// marks live objects in the heap +GC_STATIC void gc_mark(void) { + GC_DEBUG(mark, "start"); + GC_DEBUG(mark, "polarity: %d", gc_mark_polarity); + + // TODO: only if debug + // reset stats + gc_stats.live_objects = 0; + + // reset line marks + // TODO: is there a better place to do this? + for (gc_page_header_t *page = gc_last_allocated_page; page != NULL; page = page->next_page) { + for (int block_id = 0; block_id < GC_BLOCKS_PER_PAGE; block_id++) { + if (GET_BIT(page->block_bmp, block_id)) { + gc_block_header_t *block = GC_PAGE_BLOCK(page, block_id); + memset(block->line_marks, 0, GC_LINES_PER_BLOCK); + } + } + } + + // reset stacks + gc_mark_total = 0; + gc_mark_stack.pos = 0; + gc_mark_stack_next.pos = 0; + gc_mark_stack_active = &gc_mark_stack; + + GC_DEBUG_DUMP0("gc_mark.roots"); + + // mark roots + for (int i = 0; i < gc_root_count; i++) { + GC_DEBUG(mark, "root: %p", gc_roots[i]); + gc_push(*gc_roots[i], gc_roots[i]); + } + + GC_DEBUG_DUMP0("gc_mark.threads"); + + // scan threads stacks & registers + // TODO: multiple threads ... + { + hl_thread_info *t = current_thread; + void **cur = t->stack_cur; + void **top = t->stack_top; + while (cur < top) { + gc_push(*cur++, NULL); + } + cur = (void **)(&(t->gc_regs)); + top = (void **)(((char *)&(t->gc_regs)) + sizeof(jmp_buf)); + while (cur < top) { + gc_push(*cur++, NULL); + } + } + + GC_DEBUG_DUMP0("gc_mark.propagate"); + + // propagate + while (gc_mark_total-- > 0) { + gc_stats.live_objects++; + gc_object_t *p = gc_pop(); + if (p == NULL) + GC_FATAL("popped null"); + gc_metadata_t *meta = GC_METADATA(p); + + // mark line(s) + int line_id = GC_LINE_ID(p); + int words = meta->words; + gc_block_header_t *block = GC_LINE_BLOCK(p); + if (meta->medium_sized) { + gc_metadata_ext_t *meta_ext = (gc_metadata_ext_t *)meta; + words |= (meta_ext->words_ext << 4); + int last_id = GC_LINE_ID((char *)p + words * 8); + int line_span = (last_id - line_id); + GC_DEBUG(mark, "marking medium, %d lines", line_span); + memset(&block->line_marks[line_id], 1, line_span); + } else { + // for huge objects this does nothing useful + block->line_marks[line_id] = 1; + } + + if (meta->no_ptr) { + continue; + } + + // scan object + void **data = (void **)((void *)p + sizeof(gc_object_t)); + if (block->huge_sized) { + words = (GC_BLOCK_PAGE(p)->size - 8192) / 8; + GC_DEBUG(mark, "scanning huge, %d words", words); + } + // printf("%p %p %p %p -- type %s, %d words\n", p, meta, p->t, p->t->mark_bits, hl_type_name(p->t), words); + // printf("mark_bits 1 %d\n", p->t->mark_bits[0]); + // TODO: p->t->mark_bits should only be null with dynamic_mark + if (true) { //meta->dynamic_mark || p->t == NULL || p->t->mark_bits == NULL) { + for (int i = 0; i < words; i++) { + gc_push(data[i], &data[i]); + } + } else { + unsigned int *mark_bits = p->t->mark_bits; + for (int i = 0; i < words; i++) { + if (GET_BIT(mark_bits, i)) { + gc_push(data[i], &data[i]); + } + } + } + } + + GC_DEBUG_DUMP0("gc_mark.cleanup"); + + // clean up stacks + if (gc_mark_stack_active == &gc_mark_stack_next) { + free(gc_mark_stack.data); + gc_mark_stack.data = gc_mark_stack_next.data; + gc_mark_stack.capacity = gc_mark_stack_next.capacity; + gc_mark_stack_next.data = NULL; + gc_mark_stack_active = &gc_mark_stack; + } +} + +// performs a heap sweep +GC_STATIC void gc_sweep(void) { + GC_DEBUG(sweep, "start"); + + // iterate through pages + // in each page, iterate occupied (block_bmp) pages + // for each block + // - count the number of used lines + // - finalise unmarked lines ? + // - if compactable and fragmented, try compacting + // - if empty, move into global free pool + // - if not full but in full list, move into thread's recycle list + + // normal page with blocks + for (gc_page_header_t *page = gc_last_allocated_page; page != NULL; page = page->next_page) { + for (int block_id = 0; block_id < GC_BLOCKS_PER_PAGE; block_id++) { + if (GET_BIT(page->block_bmp, block_id)) { + gc_block_header_t *block = GC_PAGE_BLOCK(page, block_id); + int lines_used = 0; + for (int line_id = 0; line_id < GC_LINES_PER_BLOCK; line_id++) { + if (block->line_marks[line_id]) { + lines_used++; + } + } + GC_DEBUG_DUMP2("gc_sweep.block", block, block->line_marks); + GC_DEBUG(sweep, "block %p uses %d lines", block, lines_used); + if (lines_used == 0) { + gc_push_block(block); + } else if (lines_used < GC_LINES_PER_BLOCK && block->kind == GC_BLOCK_FULL) { + DLL_UNLINK(block); + block->kind = GC_BLOCK_RECYCLED; + // TODO: get thread by block->owner_thread + DLL_INSERT(block, current_thread->recyclable_blocks); + } + } + } + } + + // huge pages with one object + gc_page_header_t *next_page; + for (gc_page_header_t *page = gc_last_allocated_huge_page; page != NULL; page = next_page) { + next_page = page->next_page; + gc_object_t *obj = (gc_object_t *)(&((gc_block_header_t *)page)->lines[0]); + gc_metadata_t *meta = GC_METADATA(obj); + GC_DEBUG(sweep, "huge %p mark: %d %d", obj, meta->marked, gc_mark_polarity); + if (meta->marked != gc_mark_polarity) { + //gc_free_page_memory(page); + gc_push_huge(obj); + } + } + + // change polarity for the next cycle + gc_mark_polarity = 1 - gc_mark_polarity; + + gc_stats.cycles++; +} + +GC_STATIC gc_block_header_t *gc_get_block(void *p) { + if (p < gc_min_allocated || p >= gc_max_allocated || ((int_val)p & 7) != 0) { + return NULL; + } + int bucket = GC_HASH(p) % gc_page_hash.bucket_count; + for (gc_page_header_t *cur = gc_page_hash.buckets[bucket]; cur != NULL; cur = cur->next_page_bucket) { + if ((void *)p >= (void *)cur && (void *)p < (void *)((char *)cur + cur->size)) { + gc_block_header_t *ret = GC_LINE_BLOCK(p); + if ((int_val)p - (int_val)ret < 64 * GC_LINE_SIZE) { + // block header + return NULL; + } + return ret; + } + } + return NULL; +} + +HL_API void hl_blocking(bool b) { + // hl_fatal("hl_blocking not implemented"); +} + +HL_API hl_thread_info *hl_get_thread() { + return current_thread; +} + +HL_API bool hl_is_gc_ptr(void *ptr) { + return (gc_get_block(ptr) != NULL); +} + +// HL_API varray *hl_alloc_array( hl_type *t, int size ) { return NULL; } + +HL_API vdynamic *hl_alloc_dynamic( hl_type *t ) { + return (vdynamic*)hl_gc_alloc_gen(t, sizeof(vdynamic), hl_is_ptr(t) ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); +} + +static const vdynamic vdyn_true = { &hlt_bool, {true} }; +static const vdynamic vdyn_false = { &hlt_bool, {false} }; + +vdynamic *hl_alloc_dynbool( bool b ) { + return (vdynamic*)(b ? &vdyn_true : &vdyn_false); +} + +HL_API vdynamic *hl_alloc_obj( hl_type *t ) { + vobj *o; + int size; + int i; + hl_runtime_obj *rt = t->obj->rt; + if( rt == NULL || rt->methods == NULL ) rt = hl_get_obj_proto(t); + size = rt->size; + if( size & (HL_WSIZE-1) ) size += HL_WSIZE - (size & (HL_WSIZE-1)); + if( t->kind == HSTRUCT ) { + o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? GC_ALLOC_RAW : GC_ALLOC_NOPTR); + o->t = NULL; + } else { + o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); + o->t = t; + } + for(i=0;inbindings;i++) { + hl_runtime_binding *b = rt->bindings + i; + *(void**)(((char*)o) + rt->fields_indexes[b->fid]) = b->closure ? hl_alloc_closure_ptr(b->closure,b->ptr,o) : b->ptr; + } + return (vdynamic*)o; +} + +// HL_API venum *hl_alloc_enum( hl_type *t, int index ) { return NULL; } + +HL_API vvirtual *hl_alloc_virtual( hl_type *t ) { + vvirtual *v = (vvirtual*)hl_gc_alloc(t, t->virt->dataSize + sizeof(vvirtual) + sizeof(void*) * t->virt->nfields); + void **fields = (void**)(v + 1); + char *vdata = (char*)(fields + t->virt->nfields); + int i; + v->value = NULL; + v->next = NULL; + for(i=0;ivirt->nfields;i++) + fields[i] = (char*)v + t->virt->indexes[i]; + memset(vdata, 0, t->virt->dataSize); + return v; +} + +HL_API vdynobj *hl_alloc_dynobj(void) { + return (vdynobj*)hl_gc_alloc_gen(&hlt_dynobj,sizeof(vdynobj), GC_ALLOC_DYNAMIC); +} + +// HL_API vbyte *hl_alloc_bytes( int size ) { return NULL; } + +static hl_types_dump gc_types_dump = NULL; +HL_API void hl_gc_set_dump_types( hl_types_dump tdump ) { + gc_types_dump = tdump; +} + +// GC initialisation ----------------------------------------------- + +GC_STATIC void gc_init(void) { + gc_mutex_roots = hl_mutex_alloc(false); + gc_mutex_pool = hl_mutex_alloc(false); + + base_addr = (void *)0x40000000; + gc_min_allocated = (void *)0xFFFFFFFFFFFFFFFF; + gc_max_allocated = 0; + gc_last_allocated_page = NULL; + gc_last_allocated_huge_page = NULL; + gc_page_hash.bucket_count = 128; + gc_page_hash.buckets = calloc(sizeof(gc_page_header_t *), gc_page_hash.bucket_count); + gc_page_hash.total_pages = 0; + gc_pool = NULL; + gc_pool_count = 0; + gc_roots = NULL; + gc_root_count = 0; + gc_root_capacity = 0; + gc_mark_polarity = 1; + gc_mark_stack.capacity = 256; + gc_mark_stack.data = malloc(sizeof(gc_mark_stack_entry_t) * gc_mark_stack.capacity); + gc_mark_stack.pos = 0; + gc_mark_stack_next.data = NULL; + gc_mark_stack_next.pos = 0; + gc_mark_stack_next.capacity = 0; + gc_mark_stack_active = NULL; + gc_stats.live_objects = 0; + gc_stats.live_blocks = 0; + gc_stats.total_memory = 0; + gc_stats.cycles = 0; + gc_stats.total_pages = 0; + // gc_config.memory_limit = 500 * 1024 * 1024; + gc_config.memory_limit = 100 * 1024 * 1024; + gc_config.debug_fatal = true; + gc_config.debug_os = false; + gc_config.debug_page = false; + gc_config.debug_alloc = false; + gc_config.debug_mark = false; + gc_config.debug_sweep = false; + gc_config.debug_other = false; + gc_config.debug_dump = true; + gc_config.dump_file = fopen("gc.dump", "w"); +} + +GC_STATIC void gc_deinit(void) { + hl_mutex_free(gc_mutex_roots); + hl_mutex_free(gc_mutex_pool); + while (gc_last_allocated_page != NULL) { + gc_free_page_memory(gc_last_allocated_page); + } + while (gc_last_allocated_huge_page != NULL) { + gc_free_page_memory(gc_last_allocated_huge_page); + } + free(gc_page_hash.buckets); + free(gc_mark_stack.data); +} + +void hl_cache_free(); +void hl_cache_init(); + +void hl_global_init() { + gc_init(); + hl_cache_init(); +} + +void hl_global_free() { + gc_deinit(); + hl_cache_free(); +} diff --git a/src/gc.h b/src/gc.h new file mode 100644 index 000000000..f829d4180 --- /dev/null +++ b/src/gc.h @@ -0,0 +1,262 @@ +#pragma once +#include +#include +#include +#include +#include + +// OS page management ---------------------------------------------- + +// allocates a page-aligned region of memory from the OS +static void *gc_alloc_os_memory(int size); +// returns a region of memory back to the OS +static void gc_free_os_memory(void *ptr, int size); + +// GC configuration ------------------------------------------------- + +// 4 MiB pages +// 64 KiB blocks (64 blocks per page) +// 128 B lines (8KiB header + 448 lines per block) +#define GC_PAGE_SIZE (1 << 22) +#define GC_BLOCK_SIZE (1 << 16) +#define GC_LINE_SIZE (1 << 7) + +#define GC_BLOCKS_PER_PAGE 64 +#define GC_LINES_PER_BLOCK 448 + +// up to 8184 (~25% of block size) for medium objects +#define GC_MEDIUM_SIZE ((1 << 13) - 8) + +// TODO: check collisions +#define GC_HASH(ptr) (((int_val)(ptr) >> 22) ^ (((int_val)(ptr) >> 22) << 5)) + +// GC data structures ----------------------------------------------- + +typedef enum { + GC_PAGE_NORMAL = 1, + GC_PAGE_HUGE +} gc_page_kind_t; + +typedef struct gc_page_header_s { + struct gc_page_header_s *next_page; // next page in allocation order + struct gc_page_header_s *next_page_bucket; // next page in page hash bucket + unsigned char block_bmp[8]; // used blocks (for sweeping) + int size; + unsigned char free_blocks; // TODO: not really used + unsigned char kind; +} gc_page_header_t; + +typedef struct { + gc_page_header_t **buckets; + int bucket_count; + int total_pages; +} gc_page_hash_t; + +typedef struct { + unsigned char data[GC_LINE_SIZE]; +} gc_line_t; + +typedef enum { + // free: next refers to next free block (SLL) + GC_BLOCK_FREE = 1, + // new: no pointers + GC_BLOCK_NEW, + // full: prev/next refer to blocks in thread's full_blocks (DLL) + GC_BLOCK_FULL, + // recycled: prev/next refer to blocks in thread's recyclable_blocks (DLL) + GC_BLOCK_RECYCLED, + // TODO: zombie blocks + GC_BLOCK_ZOMBIE, + _force_int = 0x7FFFFFFF +} gc_block_type_t; + +typedef union { + struct { + // set to mark polarity during mark phase + uint8_t marked : 1; + // whether the object spans multiple lines + uint8_t medium_sized : 1; + // whether the object is on a page of its own + // uint8_t huge_sized : 1; + // when set, object cannot be moved + // uint8_t pinned : 1; + // whether all words of the object are potentially pointers + // (set for dynamics and pointer arrays) + uint8_t dynamic_mark : 1; + // when set, no pointers are traced in the object + uint8_t no_ptr : 1; + // number of words (sizeof(void *)) + uint8_t words : 4; + }; + uint8_t flags; +} gc_metadata_t; + +typedef union { + struct { + gc_metadata_t base; + uint8_t words_ext; + }; + uint16_t flags; +} gc_metadata_ext_t; + +typedef struct gc_block_header_s { + gc_page_header_t page_header; // only set in the first block of page + struct gc_block_header_s *prev; // meaning of prev/next depends on kind + struct gc_block_header_s *next; + gc_block_type_t kind; + int owner_thread; + int huge_sized; + unsigned char _pad[4]; + unsigned char line_marks[GC_LINES_PER_BLOCK]; + gc_metadata_t metadata[GC_LINES_PER_BLOCK * 16]; + unsigned char _pad2[512]; + gc_line_t lines[GC_LINES_PER_BLOCK]; +} gc_block_header_t; + +typedef struct gc_block_dummy_s { + unsigned char _pad[32]; + gc_block_header_t *prev; // must align with gc_block_header_t + gc_block_header_t *next; +} gc_block_dummy_t; + +typedef struct { + // bool gc_blocking; + void *lines_start; + void *lines_limit; + gc_block_dummy_t *sentinels; + gc_block_header_t *lines_block; // active block for new allocations + gc_block_header_t *full_blocks; // DLL of all full blocks, may become recycled + gc_block_header_t *recyclable_blocks; // DLL of recyclable blocks +} gc_thread_info_t; + +typedef struct { + hl_type *t; +} gc_object_t; + +typedef struct { + gc_object_t header; + void *data[]; +} gc_object_hl_t; + +typedef struct { + gc_object_t header; + hl_type *et; + int size; + int _pad; + void *elements[]; +} gc_array_t; + +typedef struct { + void *object; + void *reference; +} gc_mark_stack_entry_t; + +typedef struct { + gc_mark_stack_entry_t *data; + int pos; + int capacity; +} gc_mark_stack_t; + +struct { + unsigned long live_objects; + unsigned long live_blocks; + unsigned long total_memory; + unsigned long cycles; + int total_pages; +} gc_stats; + +struct { + unsigned long memory_limit; + bool debug_fatal; + bool debug_os; + bool debug_page; + bool debug_alloc; + bool debug_mark; + bool debug_sweep; + bool debug_other; + bool debug_dump; + FILE *dump_file; +} gc_config; + +// GC page management ---------------------------------------------- + +// allocates and initialises a GC page +static gc_page_header_t *gc_alloc_page_memory(void); +// frees a GC page +static void gc_free_page_memory(gc_page_header_t *header); + +// gets the `block`th block in `page` +#define GC_PAGE_BLOCK(page, block) ((gc_block_header_t *)((char *)(page) + (block) * GC_BLOCK_SIZE)) +// gets the page to which `block` belongs +#define GC_BLOCK_PAGE(block) ((gc_page_header_t *)((int_val)(block) & (int_val)(~(GC_PAGE_SIZE - 1)))) +// gets the block to which `line` belongs +#define GC_LINE_BLOCK(line) ((gc_block_header_t *)((int_val)(line) & (int_val)(~(GC_BLOCK_SIZE - 1)))) +// gets the index of `block` in its page +#define GC_BLOCK_ID(block) ((int)(((int_val)(block) - (int_val)GC_BLOCK_PAGE(block)) / GC_BLOCK_SIZE)) +// gets the line index of `ptr` in its block +#define GC_LINE_ID(ptr) ((int)((int_val)(ptr) - (int_val)GC_LINE_BLOCK(ptr)) / GC_LINE_SIZE - 64) + +#define GC_METADATA(obj) (&(GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024])) + +// roots ----------------------------------------------------------- + +HL_PRIM void hl_add_root(void **); +HL_PRIM void hl_remove_root(void **); + +// thread-local allocator <-> global GC +static gc_block_header_t *gc_pop_block(void); // size fixed to 4 MiB +static void gc_push_block(gc_block_header_t *); +static void *gc_pop_huge(int size); +static void gc_push_huge(void *); + +// thread setup +HL_API int hl_thread_id(void); +HL_API void hl_register_thread(void *stack_top); +HL_API void hl_unregister_thread(void); + +// thread <-> thread-local allocator + +HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags); + +HL_API void hl_gc_major(void); + +// GC internals +static void gc_stop_world(bool); + +static void gc_mark(void); +static void gc_sweep(void); // or collect, compact, copy, etc + +// marking and sweeping +static gc_block_header_t *gc_get_block(void *); // NULL if not a GC pointer + +// GC debug + +#if GC_ENABLE_DEBUG +# define GC_DEBUG(stream, f, ...) do { if (gc_config.debug_ ## stream) printf("[" #stream "] " f "\n", ##__VA_ARGS__); } while (0) +# define GC_DEBUG_DUMP_S(id) do { if (gc_config.debug_dump) { fwrite((id), 1, strlen(id) + 1, gc_config.dump_file); fflush(gc_config.dump_file); } } while (0) +# define GC_DEBUG_DUMP_R(arg) do { if (gc_config.debug_dump) { fwrite(&(arg), 1, sizeof(arg), gc_config.dump_file); fflush(gc_config.dump_file); } } while (0) +# define GC_DEBUG_DUMP0(id) do { GC_DEBUG_DUMP_S(id); int s = 0; GC_DEBUG_DUMP_R(s); } while (0) +# define GC_DEBUG_DUMP1(id, arg1) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); } while (0) +# define GC_DEBUG_DUMP2(id, arg1, arg2) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1) + sizeof(arg2); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); GC_DEBUG_DUMP_R(arg2); } while (0) +# define GC_DEBUG_DUMP3(id, arg1, arg2, arg3) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1) + sizeof(arg2) + sizeof(arg3); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); GC_DEBUG_DUMP_R(arg2); GC_DEBUG_DUMP_R(arg3); } while (0) +#else +# define GC_DEBUG(...) +# define GC_DEBUG_DUMP_S(...) +# define GC_DEBUG_DUMP_R(...) +# define GC_DEBUG_DUMP0(...) +# define GC_DEBUG_DUMP1(...) +# define GC_DEBUG_DUMP2(...) +# define GC_DEBUG_DUMP3(...) +#endif +#define GC_FATAL(msg) do { puts(msg); __builtin_trap(); } while (0) + +/* +static void gc_debug_block(gc_block_header_t *block); +static void gc_debug_general(void); +static void gc_dump(const char *); +*/ + +// GC initialisation + +static void gc_init(void); +static void gc_deinit(void); diff --git a/src/hl.h b/src/hl.h index 4824d16ab..d83010dad 100644 --- a/src/hl.h +++ b/src/hl.h @@ -686,19 +686,15 @@ HL_API void hl_tls_set( hl_tls *l, void *value ); HL_API void *hl_tls_get( hl_tls *l ); HL_API void hl_tls_free( hl_tls *l ); -// ----------------------- ALLOC -------------------------------------------------- +// ----------------------- GC ALLOC ----------------------------------------------- -#define MEM_HAS_PTR(kind) (!((kind)&2)) -#define MEM_KIND_DYNAMIC 0 -#define MEM_KIND_RAW 1 -#define MEM_KIND_NOPTR 2 -#define MEM_KIND_FINALIZER 3 -#define MEM_ALIGN_DOUBLE 128 -#define MEM_ZERO 256 +#define GC_ALLOC_DYNAMIC 1 +#define GC_ALLOC_NOPTR 2 +#define GC_ALLOC_RAW 4 HL_API void *hl_gc_alloc_gen( hl_type *t, int size, int flags ); -HL_API void hl_add_root( void *ptr ); -HL_API void hl_remove_root( void *ptr ); +HL_API void hl_add_root( void **ptr ); +HL_API void hl_remove_root( void **ptr ); HL_API void hl_gc_major( void ); HL_API bool hl_is_gc_ptr( void *ptr ); @@ -708,10 +704,14 @@ HL_API bool hl_is_blocking( void ); typedef void (*hl_types_dump)( void (*)( void *, int) ); HL_API void hl_gc_set_dump_types( hl_types_dump tdump ); -#define hl_gc_alloc_noptr(size) hl_gc_alloc_gen(&hlt_bytes,size,MEM_KIND_NOPTR) -#define hl_gc_alloc(t,size) hl_gc_alloc_gen(t,size,MEM_KIND_DYNAMIC) -#define hl_gc_alloc_raw(size) hl_gc_alloc_gen(&hlt_abstract,size,MEM_KIND_RAW) -#define hl_gc_alloc_finalizer(size) hl_gc_alloc_gen(&hlt_abstract,size,MEM_KIND_FINALIZER) +#define hl_gc_alloc_noptr(size) hl_gc_alloc_gen(&hlt_bytes, size, GC_ALLOC_NOPTR) +#define hl_gc_alloc(t, size) hl_gc_alloc_gen(t, size, GC_ALLOC_DYNAMIC) +#define hl_gc_alloc_raw(size) hl_gc_alloc_gen(&hlt_abstract, size, 0) +// TODO: MEM_KIND_RAW) +#define hl_gc_alloc_finalizer(size) hl_gc_alloc_gen(&hlt_abstract, size, 0) +// TODO: MEM_KIND_FINALIZER) + +// ----------------------- INTERNAL ALLOC ----------------------------------------- HL_API void hl_alloc_init( hl_alloc *a ); HL_API void *hl_malloc( hl_alloc *a, int size ); @@ -852,12 +852,23 @@ struct _hl_trap_ctx { #define HL_TRACK_DYNCALL 8 #define HL_TRACK_MASK (HL_TRACK_ALLOC | HL_TRACK_CAST | HL_TRACK_DYNFIELD | HL_TRACK_DYNCALL) +typedef struct gc_block_dummy_s gc_block_dummy_t; +typedef struct gc_block_header_s gc_block_header_t; + typedef struct { int thread_id; + // gc vars volatile int gc_blocking; void *stack_top; void *stack_cur; + void *lines_start; + void *lines_limit; + gc_block_dummy_t *sentinels; + gc_block_header_t *lines_block; // active block for new allocations + gc_block_header_t *full_blocks; // DLL of all full blocks, may become recycled + gc_block_header_t *recyclable_blocks; // DLL of recyclable blocks + // exception handling hl_trap_ctx *trap_current; hl_trap_ctx *trap_uncaught; diff --git a/src/memory.c b/src/memory.c new file mode 100644 index 000000000..cbb903506 --- /dev/null +++ b/src/memory.c @@ -0,0 +1,150 @@ +/* + * Copyright (C)2005-2016 Haxe Foundation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +/* + * OS-backed memory allocation. + */ + +#include "hl.h" + +#define GC_PAGE_BITS 16 +#define GC_PAGE_SIZE (1 << GC_PAGE_BITS) + +#ifdef HL_WIN +# include +# include +#else +# include +# include +#endif + +#ifdef __APPLE__ +# ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +# endif +#endif + +void *hl_alloc_executable_memory( int size ) { +#if defined(HL_WIN) && defined(HL_64) + static char *jit_address = (char*)0x000076CA9F000000; + void *ptr; +retry_jit_alloc: + ptr = VirtualAlloc(jit_address,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE); + if( !ptr ) { + jit_address = (char*)(((int_val)jit_address)>>1); // fix for Win7 - will eventually reach NULL + goto retry_jit_alloc; + } + jit_address += size + ((-size) & (GC_PAGE_SIZE - 1)); + return ptr; +#elif defined(HL_WIN) + void *ptr = VirtualAlloc(NULL,size,MEM_RESERVE|MEM_COMMIT,PAGE_EXECUTE_READWRITE); + return ptr; +#elif defined(HL_CONSOLE) + return NULL; +#else + void *p; + p = mmap(NULL,size,PROT_READ|PROT_WRITE|PROT_EXEC,(MAP_PRIVATE|MAP_ANONYMOUS),-1,0); + return p; +#endif +} + +void hl_free_executable_memory( void *c, int size ) { +#if defined(HL_WIN) + VirtualFree(c,0,MEM_RELEASE); +#elif !defined(HL_CONSOLE) + munmap(c, size); +#endif +} + +#if defined(HL_CONSOLE) +void *sys_alloc_align( int size, int align ); +void sys_free_align( void *ptr, int size ); +#elif !defined(HL_WIN) +static void *base_addr = (void*)0x40000000; +#endif + +void *gc_alloc_page_memory( int size ) { +#if defined(HL_WIN) +# if defined(GC_DEBUG) && defined(HL_64) +# define STATIC_ADDRESS +# endif +# ifdef STATIC_ADDRESS + // force out of 32 bits addresses to check loss of precision + static char *start_address = (char*)0x100000000; +# else + static void *start_address = NULL; +# endif + void *ptr = VirtualAlloc(start_address,size,MEM_RESERVE|MEM_COMMIT,PAGE_READWRITE); +# ifdef STATIC_ADDRESS + if( ptr == NULL && start_address ) { + start_address = NULL; + return gc_alloc_page_memory(size); + } + start_address += size + ((-size) & (GC_PAGE_SIZE - 1)); +# endif + return ptr; +#elif defined(HL_CONSOLE) + return sys_alloc_align(size, GC_PAGE_SIZE); +#else + /* + int i = 0; + while( gc_will_collide(base_addr,size) ) { + base_addr = (char*)base_addr + GC_PAGE_SIZE; + i++; + // most likely our hashing creates too many collisions + if( i >= 1 << (GC_LEVEL0_BITS + GC_LEVEL1_BITS + 2) ) + return NULL; + } + */ + void *ptr = mmap(base_addr,size,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + if( ptr == (void*)-1 ) + return NULL; + if( ((int_val)ptr) & (GC_PAGE_SIZE-1) ) { + munmap(ptr,size); + void *tmp; + int tmp_size = (int)((int_val)ptr - (int_val)base_addr); + if( tmp_size > 0 ) { + base_addr = (void*)((((int_val)ptr) & ~(GC_PAGE_SIZE-1)) + GC_PAGE_SIZE); + tmp = ptr; + } else { + base_addr = (void*)(((int_val)ptr) & ~(GC_PAGE_SIZE-1)); + tmp = NULL; + } + if( tmp ) tmp = mmap(tmp,tmp_size,PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0); + ptr = gc_alloc_page_memory(size); + if( tmp ) munmap(tmp,tmp_size); + return ptr; + } + base_addr = (char*)ptr+size; + return ptr; +#endif +} + +void gc_free_page_memory( void *ptr, int size ) { +#ifdef HL_WIN + VirtualFree(ptr, 0, MEM_RELEASE); +#elif defined(HL_CONSOLE) + sys_free_align(ptr,size); +#else + munmap(ptr,size); +#endif +} diff --git a/src/std/array.c b/src/std/array.c index 429c24336..7b7f7f327 100644 --- a/src/std/array.c +++ b/src/std/array.c @@ -25,8 +25,7 @@ HL_PRIM varray *hl_alloc_array( hl_type *at, int size ) { int esize = hl_type_size(at); varray *a; if( size < 0 ) hl_error("Invalid array size"); - a = (varray*)hl_gc_alloc_gen(&hlt_array, sizeof(varray) + esize*size, (hl_is_ptr(at) ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR) | MEM_ZERO); - a->t = &hlt_array; + a = (varray*)hl_gc_alloc_gen(&hlt_array, sizeof(varray) + esize*size, hl_is_ptr(at) ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); a->at = at; a->size = size; return a; diff --git a/src/std/types.c b/src/std/types.c index 26581927d..50f964251 100644 --- a/src/std/types.c +++ b/src/std/types.c @@ -530,8 +530,7 @@ HL_PRIM bool hl_type_enum_eq( venum *a, venum *b ) { HL_PRIM venum *hl_alloc_enum( hl_type *t, int index ) { hl_enum_construct *c = t->tenum->constructs + index; - venum *v = (venum*)hl_gc_alloc_gen(t, c->size, MEM_KIND_DYNAMIC | (c->hasptr ? 0 : MEM_KIND_NOPTR) | MEM_ZERO); - v->t = t; + venum *v = (venum*)hl_gc_alloc_gen(t, c->size, c->hasptr ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); v->index = index; return v; } From f2def2515384a84d5a135188f8aba660e3751966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Tue, 31 Mar 2020 15:56:30 +0100 Subject: [PATCH 02/13] move hlgc tests into repo --- other/gctests/Makefile | 3 + other/gctests/bench/mandelbrot.c | 142 +++++++++++++++ other/gctests/main.c | 291 +++++++++++++++++++++++++++++++ other/gctests/suite.h | 47 +++++ other/gctests/utils.h | 41 +++++ src/gc.c | 114 ++++++------ src/gc.h | 40 +++-- 7 files changed, 607 insertions(+), 71 deletions(-) create mode 100644 other/gctests/Makefile create mode 100644 other/gctests/bench/mandelbrot.c create mode 100644 other/gctests/main.c create mode 100644 other/gctests/suite.h create mode 100644 other/gctests/utils.h diff --git a/other/gctests/Makefile b/other/gctests/Makefile new file mode 100644 index 000000000..41ba9c80c --- /dev/null +++ b/other/gctests/Makefile @@ -0,0 +1,3 @@ +main: main.c ../../src/gc.h bench/mandelbrot.c + gcc -Wall -Werror -pedantic -Wno-unused-function -Wno-unused-variable -Wno-gnu-zero-variadic-macro-arguments \ + -O3 -std=c11 -o main -I ../../src main.c bench/mandelbrot.c -lhl diff --git a/other/gctests/bench/mandelbrot.c b/other/gctests/bench/mandelbrot.c new file mode 100644 index 000000000..3ccecd8f4 --- /dev/null +++ b/other/gctests/bench/mandelbrot.c @@ -0,0 +1,142 @@ +#include "hl.h" +#include "gc.h" +#include "../utils.h" +#include "../suite.h" + + +#ifdef WIN32 + +#include +static double hires_timestamp(void) { + LARGE_INTEGER t, f; + QueryPerformanceCounter(&t); + QueryPerformanceFrequency(&f); + return (double)t.QuadPart / (double)f.QuadPart; +} + +#else + +#include +#include +static double hires_timestamp(void) { + struct timeval t; + struct timezone tzp; + gettimeofday(&t, &tzp); + return t.tv_sec + t.tv_usec * 1e-6; +} + +#endif + +TEST_TYPE(rgb, 3, {0}, { + int r; + int g; + int b; +}); +TEST_TYPE(complex, 3, {0}, { + double i; + double j; +}); + +#define MB_SIZE 25 +#define MB_MAX_ITERATIONS 1000 +#define MB_MAX_RAD (1 << 16) +#define MB_WIDTH (35 * MB_SIZE) +#define MB_HEIGHT (20 * MB_SIZE) + +static hlt_rgb_t *createPalette(double inFraction) { + hlt_rgb_t *ret = (hlt_rgb_t *)hl_alloc_obj(&hlt_rgb); + ret->r = inFraction * 255; + ret->g = (1.0 - inFraction) * 255; + ret->b = (0.5 - abs(inFraction - 0.5)) * 2 * 255; + return ret; +} + +static hlt_complex_t *createComplex(double inI, double inJ) { + hlt_complex_t *ret = (hlt_complex_t *)hl_alloc_obj(&hlt_complex); + ret->i = inI; + ret->j = inJ; + return ret; +} + +static double complexLength2(hlt_complex_t *val) { + return val->i * val->i + val->j * val->j; +} + +static hlt_complex_t *complexAdd(hlt_complex_t *val0, hlt_complex_t *val1) { + return createComplex(val0->i + val1->i, val0->j + val1->j); +} + +static hlt_complex_t *complexSquare(hlt_complex_t *val) { + return createComplex(val->i * val->i - val->j * val->j, 2.0 * val->i * val->j); +} + +BEGIN_BENCH_CASE(mandelbrot) { + varray *palette = hl_alloc_array(&hlt_rgb, MB_MAX_ITERATIONS + 1); + hl_add_root((void *)&palette); + for (int i = 0; i < MB_MAX_ITERATIONS + 1; i++) { + hl_aptr(palette, hlt_rgb_t *)[i] = createPalette((double)i / MB_MAX_ITERATIONS); + } + + /* + for (int i = 0; i < MB_SIZE; i++) { + printf("%d %d %d\n", palette->data[i]->r, palette->data[i]->g, palette->data[i]->b); + } + */ + + varray *image = hl_alloc_array(&hlt_rgb, MB_WIDTH * MB_HEIGHT); + hl_add_root((void *)&image); + int outPixel = 0; + double scale = 0.1 / MB_SIZE; + + hlt_complex_t *offset = NULL; + hlt_complex_t *val = NULL; + hl_add_root((void *)&offset); + hl_add_root((void *)&val); + + for (int y = 0; y < MB_HEIGHT; y++) { + if ((y % 10) == 0) { + printf("%d: %d pages, %lu live objects, %lu live blocks, %lu bytes used, %lu collection cycles\n", y, gc_stats->total_pages, gc_stats->live_objects, gc_stats->live_blocks, gc_stats->total_memory, gc_stats->cycles); + } + for (int x = 0; x < MB_WIDTH; x++) { + int iteration = 0; + + // "reduce_allocs" + /* + double offsetI = x * scale - 2.5; + double offsetJ = y * scale - 1.0; + double valI = 0.0; + double valJ = 0.0; + while (valI * valI + valJ * valJ < MB_MAX_RAD && iteration < MB_MAX_ITERATIONS) { + double vi = valI; + valI = valI * valI - valJ * valJ + offsetI; + valJ = 2.0 * vi * valJ + offsetJ; + iteration++; + } + */ + // normal + offset = createComplex(x * scale - 2.5, y * scale - 1); + val = createComplex(0.0, 0.0); + while (complexLength2(val) < MB_MAX_RAD && iteration < MB_MAX_ITERATIONS) { + val = complexSquare(val); + val = complexAdd(val, offset); + iteration++; + } + + hl_aptr(image, hlt_rgb_t *)[outPixel++] = hl_aptr(palette, hlt_rgb_t *)[iteration]; + } + } + + FILE *f = fopen("mandelbrot.ppm", "w"); + fprintf(f, "P6 %d %d 255\n", MB_WIDTH, MB_HEIGHT); + for (int i = 0; i < MB_WIDTH * MB_HEIGHT; i++) { + hlt_rgb_t *pixel = hl_aptr(image, hlt_rgb_t *)[i]; + // printf("%d %p\n", i, pixel); + unsigned char r = pixel->r & 0xFF; + unsigned char g = pixel->g & 0xFF; + unsigned char b = pixel->b & 0xFF; + fwrite(&r, 1, 1, f); + fwrite(&g, 1, 1, f); + fwrite(&b, 1, 1, f); + } + fclose(f); +} END_BENCH_CASE diff --git a/other/gctests/main.c b/other/gctests/main.c new file mode 100644 index 000000000..4c4f76673 --- /dev/null +++ b/other/gctests/main.c @@ -0,0 +1,291 @@ +#include +#include +#include "hl.h" +#include "gc.h" +#include "utils.h" +#include "suite.h" + +// types + +TEST_TYPE(one_ptr, 1, {1}, { + void *ptr; +}); +TEST_TYPE(some_ptr, 10, {1 | 4 | 64 | 128 | 512}, { + void *ptr_0; + void *data_1; + void *ptr_2; + void *data_3; + void *data_4; + void *data_5; + void *ptr_6; + void *ptr_7; + void *data_8; + void *ptr_9; +}); +TEST_TYPE(medium_one_ptr, 256, {1}, { + void *ptr; + void *data[255]; +}); +TEST_TYPE(big_no_ptr, 10000, {0}, { + int _dummy; +}); + +// test cases + +BEGIN_TEST_CASE(sanity) { + ASSERT(sizeof(hl_type *) == 8); + ASSERT(sizeof(gc_page_header_t) == 32); + ASSERT(sizeof(gc_metadata_t) == 1); + ASSERT(sizeof(gc_metadata_ext_t) == 2); + ASSERT(sizeof(gc_block_header_t) == GC_BLOCK_SIZE); + ASSERT(offsetof(gc_block_header_t, lines) == 64 * GC_LINE_SIZE); + + ASSERT(GC_PAGE_BLOCK(0x686178650B400000ul, 0) == (gc_block_header_t *)0x686178650B400000ul); + ASSERT(GC_PAGE_BLOCK(0x686178650B400000ul, 13) == (gc_block_header_t *)0x686178650B4D0000ul); + ASSERT(GC_PAGE_BLOCK(0x686178650B400000ul, 63) == (gc_block_header_t *)0x686178650B7F0000ul); + + ASSERT(GC_BLOCK_ID(0x686178650B400000ul) == 0); + ASSERT(GC_BLOCK_ID(0x686178650B4D0000ul) == 13); + ASSERT(GC_BLOCK_ID(0x686178650B7F0000ul) == 63); + + ASSERT(GC_BLOCK_PAGE(0x686178650B400000ul) == (gc_page_header_t *)0x686178650B400000ul); + ASSERT(GC_BLOCK_PAGE(0x686178650B400123ul) == (gc_page_header_t *)0x686178650B400000ul); + ASSERT(GC_BLOCK_PAGE(0x686178650B4D0000ul) == (gc_page_header_t *)0x686178650B400000ul); + ASSERT(GC_BLOCK_PAGE(0x686178650B7F0000ul) == (gc_page_header_t *)0x686178650B400000ul); + + ASSERT(GC_LINE_BLOCK(0x686178650B4D0000ul) == (gc_block_header_t *)0x686178650B4D0000ul); + ASSERT(GC_LINE_BLOCK(0x686178650B4D0080ul) == (gc_block_header_t *)0x686178650B4D0000ul); + ASSERT(GC_LINE_BLOCK(0x686178650B4D2000ul) == (gc_block_header_t *)0x686178650B4D0000ul); + ASSERT(GC_LINE_BLOCK(0x686178650B4D2123ul) == (gc_block_header_t *)0x686178650B4D0000ul); + ASSERT(GC_LINE_BLOCK(0x686178650B4DFF80ul) == (gc_block_header_t *)0x686178650B4D0000ul); + + ASSERT(GC_LINE_ID(0x686178650B4D2000ul) == 0); + ASSERT(GC_LINE_ID(0x686178650B4D2123ul) == 2); + ASSERT(GC_LINE_ID(0x686178650B4DFF80ul) == 447); + + ASSERT(GC_METADATA(0x686178650B4D2000ul) == (gc_metadata_t *)0x686178650B4D0200ul); + ASSERT(GC_METADATA(0x686178650B4D2010ul) == (gc_metadata_t *)0x686178650B4D0202ul); + ASSERT(GC_METADATA(0x686178650B4D2018ul) == (gc_metadata_t *)0x686178650B4D0203ul); + ASSERT(GC_METADATA(0x686178650B4D2128ul) == (gc_metadata_t *)0x686178650B4D0225ul); + ASSERT(GC_METADATA(0x686178650B4DFFF0ul) == (gc_metadata_t *)0x686178650B4D1DFEul); +} END_TEST_CASE + +// a cycle of one object +BEGIN_TEST_CASE(self_cycle) { + hlt_one_ptr_t *obj = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + obj->ptr = obj; + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 1); + + printf("%p\n", obj); +} END_TEST_CASE + +// a cycle of three objects +BEGIN_TEST_CASE(tiny_cycle) { + hlt_one_ptr_t *obj_1 = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hlt_one_ptr_t *obj_2 = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hlt_one_ptr_t *obj_3 = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + + obj_1->ptr = obj_2; + obj_2->ptr = obj_3; + obj_3->ptr = obj_1; + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 3); + ASSERT(gc_stats->total_pages == 1); + hl_gc_major(); + ASSERT(gc_stats->live_objects == 3); +} END_TEST_CASE + +// a cycle spanning more than one block +BEGIN_TEST_CASE(block_cycle) { + hlt_one_ptr_t *first = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hlt_one_ptr_t *last = first; + + for (int i = 0; i < 5000; i++) { + hlt_one_ptr_t *next = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + last->ptr = next; + last = next; + } + last->ptr = first; + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 5001); +} END_TEST_CASE + +// a cycle in objects with many pointers +BEGIN_TEST_CASE(complex_cycle) { + hlt_some_ptr_t *objs[100]; + for (int i = 0; i < 100; i++) { + objs[i] = (hlt_some_ptr_t *)hl_alloc_obj(&hlt_some_ptr); + } + + for (int i = 0; i < 100; i++) { + objs[i]->ptr_0 = objs[(i + 1 + 0) % 100]; + objs[i]->ptr_2 = objs[(i + 1 + 2) % 100]; + objs[i]->ptr_6 = objs[(i + 1 + 6) % 100]; + objs[i]->ptr_7 = objs[(i + 1 + 7) % 100]; + objs[i]->ptr_9 = objs[(i + 1 + 9) % 100]; + } + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 100); +} END_TEST_CASE + +// allocate small objects and fit a medium object +BEGIN_TEST_CASE(medium_object_recycle) { + // start with small objects + // BLOCK 1 | + // s > s > s > s > ... > s > s > ... > s | + // create gap large enough for a medium object + // BLOCK 1 | + // s > s > s - ... - s > ... > s | + // \________________^ + // allocate a full block of dummy objects + // BLOCK 1 | BLOCK 2 | + // s > s > s - ... - s > ... > s | s s s s ... | + // \________________^ + // allocate a small object, forces recycle of block 1 + // BLOCK 1 | + // s > s > s s ... - s > ... > s | + // \________________^ + // allocate a medium object, should fit in block 1 + // BLOCK 1 | + // s > s > s > s > [m .....] s > ... > s | + // \________^ + + // (448 lines * 128 bytes per line) / 16 bytes per object = 3584 + #define OBJ_COUNT 3584 + + hlt_one_ptr_t *objs[OBJ_COUNT]; + for (int i = 0; i < OBJ_COUNT; i++) { + objs[i] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + } + + for (int i = 0; i < OBJ_COUNT - 1; i++) { + objs[i]->ptr = objs[(i + 1) % OBJ_COUNT]; + } + + hl_gc_major(); + ASSERT(gc_stats->live_objects == OBJ_COUNT); + + // a gap of 129 small objects is the minimum to fit the medium one + ASSERT(sizeof(hlt_one_ptr_t) == 16); + ASSERT(sizeof(hlt_medium_one_ptr_t) == 2056); + // ceil(2056 / 16) = 129 + // add 25 more to allow for: + // - worst-case conservatively marked lines on the "left" + // - worst-case occupied line on the "right" + int gap = 129 + 25; + objs[1337]->ptr = objs[1338 + gap]; + for (int i = 1338; i < 1338 + gap; i++) { + objs[i] = NULL; + } + hl_gc_major(); + ASSERT(gc_stats->live_objects == OBJ_COUNT - gap); + + // exhaust full block to force recycle of first block + for (int i = 0; i < OBJ_COUNT; i++) { + hl_alloc_obj(&hlt_one_ptr); + } + + hl_gc_major(); + ASSERT(gc_stats->live_objects == OBJ_COUNT - gap); + + objs[1338] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hlt_medium_one_ptr_t *med = (hlt_medium_one_ptr_t *)hl_alloc_obj(&hlt_medium_one_ptr); + objs[1337]->ptr = objs[1338]; + objs[1338]->ptr = med; + med->ptr = objs[1338 + gap]; + hl_gc_major(); + ASSERT(gc_stats->live_objects == OBJ_COUNT - gap + 2); + ASSERT(gc_get_block(objs[0]) == gc_get_block(objs[1338])); + ASSERT(gc_get_block(objs[0]) == gc_get_block(med)); +} END_TEST_CASE + +BEGIN_TEST_CASE(big_object) { + hlt_one_ptr_t *obj = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + + hlt_big_no_ptr_t *big = (hlt_big_no_ptr_t *)hl_alloc_obj(&hlt_big_no_ptr); + obj->ptr = big; + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 2); + ASSERT(gc_stats->total_pages == 2); + + printf("%p %p\n", obj, &obj); // force stack variable for obj + printf("%p %p\n", big, &big); // force stack variable for big +} END_TEST_CASE + +BEGIN_TEST_CASE(simple_array) { + varray *arr = hl_alloc_array(&hlt_one_ptr, 20); + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 1); + + for (int i = 0; i < 20; i++) { + hlt_one_ptr_t *obj = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hl_aptr(arr, hlt_one_ptr_t *)[i] = obj; + obj = NULL; + } + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 21); + + for (int i = 0; i < 20; i++) { + hlt_one_ptr_t *obj = hl_aptr(arr, hlt_one_ptr_t *)[i]; + obj->ptr = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + obj = NULL; + } + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 41); +} END_TEST_CASE + +BEGIN_TEST_CASE(big_array) { + varray *arr = hl_alloc_array(&hlt_one_ptr, 15000); + //hl_add_root((void *)&arr); + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 1); + + hl_aptr(arr, hlt_one_ptr_t *)[0] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hl_aptr(arr, hlt_one_ptr_t *)[0] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hl_aptr(arr, hlt_one_ptr_t *)[0] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hl_aptr(arr, hlt_one_ptr_t *)[0] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + hl_aptr(arr, hlt_one_ptr_t *)[14999] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 3); + + printf("%p %p\n", arr, &arr); // force stack variable for arr +} END_TEST_CASE + +// benchmark cases + +void bench_mandelbrot(int); + +// test runner + +int main(int argc, char **argv) { + int assertions = 0; + int assertions_successful = 0; + test_sanity(&assertions, &assertions_successful); + test_self_cycle(&assertions, &assertions_successful); + test_tiny_cycle(&assertions, &assertions_successful); + test_block_cycle(&assertions, &assertions_successful); + test_complex_cycle(&assertions, &assertions_successful); + test_medium_object_recycle(&assertions, &assertions_successful); + test_big_object(&assertions, &assertions_successful); + test_simple_array(&assertions, &assertions_successful); + test_big_array(&assertions, &assertions_successful); + puts("---"); + puts("TOTAL:"); + printf(" %d / %d checks passed\n", assertions_successful, assertions); + + if (assertions_successful != assertions) + return 1; + + bench_mandelbrot(1); +} diff --git a/other/gctests/suite.h b/other/gctests/suite.h new file mode 100644 index 000000000..ff45f99dc --- /dev/null +++ b/other/gctests/suite.h @@ -0,0 +1,47 @@ +#pragma once + +#define BEGIN_TEST_CASE(name) \ + void test_ ## name (int *_total_assertions, int *_total_assertions_successful) { \ + puts("TEST: " # name); \ + gc_stats_t *gc_stats = gc_init(); \ + hl_register_thread(_total_assertions); \ + const char *_test_name = #name; \ + int _assertions = 0; \ + int _assertions_successful = 0; \ + do +#define END_TEST_CASE \ + while (0); \ + printf("%d / %d checks passed\n", _assertions_successful, _assertions); \ + hl_unregister_thread(); \ + gc_deinit(); \ + } + +#define BEGIN_BENCH_CASE(name) \ + void bench_ ## name (int _bench_count) { \ + puts("BENCH: " # name); \ + gc_stats_t *gc_stats = gc_init(); \ + hl_register_thread(&gc_stats); \ + const char *_bench_name = #name; \ + double _bench_start = hires_timestamp(); \ + for (int _bench_num = 0; _bench_num < _bench_count; _bench_num++) +#define END_BENCH_CASE \ + double _bench_end = hires_timestamp(); \ + hl_unregister_thread(); \ + gc_deinit(); \ + printf("\x1B[38;5;33m[.] benchmark %s completed in %.02f seconds\n\x1B[0m", _bench_name, _bench_end - _bench_start); \ + } + +#define ASSERT(cond) \ + _assertions++; \ + (*_total_assertions)++; \ + if (cond) { \ + _assertions_successful++; \ + (*_total_assertions_successful)++; \ + printf("\x1B[38;5;28m[.] assertion passed in %s (line %d): " #cond "\n\x1B[0m", _test_name, __LINE__); \ + } else { \ + printf("\x1B[38;5;160m[!] assertion failed in %s (line %d): " #cond "\n\x1B[0m", _test_name, __LINE__); \ + printf("%d / %d checks passed\n", _assertions_successful, _assertions); \ + hl_unregister_thread(); \ + gc_deinit(); \ + return; \ + } diff --git a/other/gctests/utils.h b/other/gctests/utils.h new file mode 100644 index 000000000..3078e42cf --- /dev/null +++ b/other/gctests/utils.h @@ -0,0 +1,41 @@ +#pragma once + +// branch optimisation +#define LIKELY(c) __builtin_expect((c), 1) +#define UNLIKELY(c) __builtin_expect((c), 0) + +// bit access +#define GET_BIT(bmp, bit) ((bmp)[(bit) >> 3] & (1 << ((bit) & 7))) +#define SET_BIT0(bmp, bit) (bmp)[(bit) >> 3] &= 0xFF ^ (1 << ((bit) & 7)); +#define SET_BIT1(bmp, bit) (bmp)[(bit) >> 3] |= 1 << ((bit) & 7); +#define SET_BIT(bmp, bit, val) (bmp)[(bit) >> 3] = ((bmp)[(bit) >> 3] & (0xFF ^ (1 << ((bit) & 7)))) | ((val) << ((bit) & 7)); + +// doubly-linked lists +#define DLL_UNLINK(obj) do { \ + (obj)->next->prev = (obj)->prev; \ + (obj)->prev->next = (obj)->next; \ + (obj)->next = NULL; \ + (obj)->prev = NULL; \ + } while (0) +#define DLL_INSERT(obj, head) do { \ + (obj)->next = (head)->next; \ + (obj)->prev = (head); \ + (head)->next = (obj); \ + (obj)->next->prev = (obj); \ + } while (0) + +// HL type stubbing +#define TEST_TYPE(name, words, bits, fields) \ + hl_type hlt_ ## name; \ + hl_runtime_obj hltr_ ## name = { \ + .t = &hlt_ ## name, .nfields = words, .nproto = 0, .size = (1 + words) * sizeof(void *), \ + .nmethods = 0, .nbindings = 0, .hasPtr = true, .methods = NULL, \ + .fields_indexes = NULL, .bindings = NULL, .parent = NULL, .toStringFun = NULL, \ + .compareFun = NULL, .castFun = NULL, .getFieldFun = NULL, .nlookup = 0, .lookup = NULL }; \ + hl_type_obj hlto_ ## name = { .rt = &hltr_ ## name }; \ + unsigned int hltm_ ## name[(words + 31) / 32] = bits; \ + hl_type hlt_ ## name = { HOBJ, .obj = &hlto_ ## name, .mark_bits = hltm_ ## name }; \ + typedef struct { \ + hl_type *t; \ + struct fields; \ + } hlt_ ## name ## _t diff --git a/src/gc.c b/src/gc.c index 0ab374167..70ff318c6 100644 --- a/src/gc.c +++ b/src/gc.c @@ -23,8 +23,6 @@ #error "no threads with gc for now" #endif -#define GC_STATIC - // utilities -------------------------------------------------------- // branch optimisation @@ -51,6 +49,11 @@ (obj)->next->prev = (obj); \ } while (0) +// state structs + +static gc_stats_t *gc_stats; +static gc_config_t *gc_config; + // OS page management ---------------------------------------------- static void *base_addr; @@ -58,9 +61,9 @@ static void *base_addr; // allocates a page-aligned region of memory from the OS GC_STATIC void *gc_alloc_os_memory(int size) { GC_DEBUG_DUMP1("gc_alloc_os_memory.enter", size); - if (gc_stats.total_memory + size >= gc_config.memory_limit) { + if (gc_stats->total_memory + size >= gc_config->memory_limit) { GC_DEBUG_DUMP0("gc_alloc_os_memory.fail.oom"); - GC_DEBUG(fatal, "using %lu / %lu bytes, need %d more", gc_stats.total_memory, gc_config.memory_limit, size); + GC_DEBUG(fatal, "using %lu / %lu bytes, need %d more", gc_stats->total_memory, gc_config->memory_limit, size); GC_FATAL("OOM: memory limit hit"); } void *ptr = mmap(base_addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); @@ -68,7 +71,7 @@ GC_STATIC void *gc_alloc_os_memory(int size) { GC_DEBUG_DUMP0("gc_alloc_os_memory.fail.other"); GC_FATAL("failed to allocate page"); } - gc_stats.total_memory += size; + gc_stats->total_memory += size; if (((int_val)ptr) & (GC_PAGE_SIZE - 1)) { // re-align munmap(ptr,size); @@ -98,7 +101,7 @@ GC_STATIC void *gc_alloc_os_memory(int size) { // returns a region of memory back to the OS GC_STATIC void gc_free_os_memory(void *ptr, int size) { munmap(ptr, size); - gc_stats.total_memory -= size; + gc_stats->total_memory -= size; GC_DEBUG_DUMP2("gc_free_os_memory.success", ptr, size); GC_DEBUG(os, "reclaimed OS page: %p", ptr); } @@ -125,7 +128,7 @@ GC_STATIC void gc_add_page(gc_page_header_t *header) { gc_max_allocated = (char *)header + GC_PAGE_SIZE; } - gc_stats.total_pages++; + gc_stats->total_pages++; gc_page_hash.total_pages++; double load_factor = gc_page_hash.total_pages / gc_page_hash.bucket_count; if (load_factor > 0.75) { // TODO: limit the growth of buckets @@ -211,16 +214,16 @@ GC_STATIC void gc_free_page_memory(gc_page_header_t *header) { } int bucket = GC_HASH(header) % gc_page_hash.bucket_count; last = &gc_page_hash.buckets[bucket]; - for (gc_page_header_t *cur = gc_page_hash.buckets[bucket]; cur != NULL; cur = cur->next_page_bucket) { + for (gc_page_header_t *cur = *last; cur != NULL; cur = cur->next_page_bucket) { if (cur == header) { *last = cur->next_page_bucket; break; } - last = &cur->next_page; + last = &cur->next_page_bucket; } // TODO: maybe rearrange if load factor is low? gc_page_hash.total_pages--; - gc_stats.total_pages--; + gc_stats->total_pages--; gc_free_os_memory(header, header->size); GC_DEBUG_DUMP1("gc_free_page_memory.success", header); } @@ -386,9 +389,9 @@ GC_STATIC void gc_stop_world( bool b ) { GC_STATIC gc_block_header_t *gc_pop_block(void) { hl_mutex_acquire(gc_mutex_pool); // TODO: mutexes need to be re-entrant, or lock later if (UNLIKELY(gc_pool_count == 0)) { - // if (gc_stats.total_memory * 1.2 >= gc_config.memory_limit) { - if (gc_stats.live_blocks > 0) { - GC_DEBUG(alloc, "using %lu / %lu bytes", gc_stats.total_memory, gc_config.memory_limit); + // if (gc_stats->total_memory * 1.2 >= gc_config->memory_limit) { + if (gc_stats->live_blocks > 0) { + GC_DEBUG(alloc, "using %lu / %lu bytes", gc_stats->total_memory, gc_config->memory_limit); // GC_FATAL("OOM: memory limit hit"); GC_DEBUG(alloc, "memory limit hit, triggering major ..."); hl_gc_major(); @@ -398,9 +401,6 @@ GC_STATIC gc_block_header_t *gc_pop_block(void) { } } gc_block_header_t *block = gc_pool; - if ((((int_val)block->next) & (int_val)(0x10000 - 1)) != 0) { - GC_FATAL("pool corrupted in pop"); - } gc_pool = block->next; gc_page_header_t *page = GC_BLOCK_PAGE(block); // TODO: mutex per page? @@ -412,7 +412,7 @@ GC_STATIC gc_block_header_t *gc_pop_block(void) { block->owner_thread = hl_thread_id(); GC_DEBUG(alloc, "got block %p", block); GC_DEBUG_DUMP1("gc_pop_block.success", block); - gc_stats.live_blocks++; + gc_stats->live_blocks++; return block; } @@ -431,12 +431,9 @@ GC_STATIC void gc_push_block(gc_block_header_t *block) { SET_BIT0(page->block_bmp, GC_BLOCK_ID(block)); // TODO: manage disorder in GC pool (prefer address order) block->next = gc_pool; - if ((((int_val)block) & (int_val)(0x10000 - 1)) != 0) { - GC_FATAL("pool corrupted in push"); - } gc_pool = block; gc_pool_count++; - gc_stats.live_blocks--; + gc_stats->live_blocks--; hl_mutex_release(gc_mutex_pool); GC_DEBUG_DUMP1("gc_push_block.success", block); } @@ -457,7 +454,7 @@ GC_STATIC void gc_push_huge(void *huge) { gc_page_header_t *header = GC_BLOCK_PAGE(huge); GC_DEBUG(alloc, "freed huge %p", huge); GC_DEBUG_DUMP1("gc_push_huge.success", header); - gc_free_os_memory(header, header->size); + gc_free_page_memory(header); } // thread <-> thread-local allocator ------------------------------- @@ -615,8 +612,8 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { // triggers a major GC collection HL_API void hl_gc_major(void) { - GC_DEBUG_DUMP0("hl_gc_major.entry"); gc_stop_world(true); + GC_DEBUG_DUMP0("hl_gc_major.entry"); gc_mark(); gc_sweep(); gc_stop_world(false); @@ -633,6 +630,7 @@ GC_STATIC void gc_push(void *p, void *ref) { if (gc_mark_stack_active == &gc_mark_stack) { GC_DEBUG(mark, "mark stack growth alloc"); // ran out of space on original stack, allocate a new one and start using it + gc_mark_stack_next.pos = 0; gc_mark_stack_next.capacity = gc_mark_stack.capacity << 1; gc_mark_stack_next.data = malloc(sizeof(gc_mark_stack_entry_t) * gc_mark_stack_next.capacity); gc_mark_stack_active = &gc_mark_stack_next; @@ -694,7 +692,7 @@ GC_STATIC void gc_mark(void) { // TODO: only if debug // reset stats - gc_stats.live_objects = 0; + gc_stats->live_objects = 0; // reset line marks // TODO: is there a better place to do this? @@ -743,7 +741,7 @@ GC_STATIC void gc_mark(void) { // propagate while (gc_mark_total-- > 0) { - gc_stats.live_objects++; + gc_stats->live_objects++; gc_object_t *p = gc_pop(); if (p == NULL) GC_FATAL("popped null"); @@ -770,7 +768,7 @@ GC_STATIC void gc_mark(void) { } // scan object - void **data = (void **)((void *)p + sizeof(gc_object_t)); + void **data = (void **)((void *)p + sizeof(hl_type *)); if (block->huge_sized) { words = (GC_BLOCK_PAGE(p)->size - 8192) / 8; GC_DEBUG(mark, "scanning huge, %d words", words); @@ -779,12 +777,12 @@ GC_STATIC void gc_mark(void) { // printf("mark_bits 1 %d\n", p->t->mark_bits[0]); // TODO: p->t->mark_bits should only be null with dynamic_mark if (true) { //meta->dynamic_mark || p->t == NULL || p->t->mark_bits == NULL) { - for (int i = 0; i < words; i++) { + for (int i = 0; i < words - 1; i++) { // skip hl_type *t gc_push(data[i], &data[i]); } } else { unsigned int *mark_bits = p->t->mark_bits; - for (int i = 0; i < words; i++) { + for (int i = 0; i < words - 1; i++) { // skip hl_type *t if (GET_BIT(mark_bits, i)) { gc_push(data[i], &data[i]); } @@ -858,7 +856,7 @@ GC_STATIC void gc_sweep(void) { // change polarity for the next cycle gc_mark_polarity = 1 - gc_mark_polarity; - gc_stats.cycles++; + gc_stats->cycles++; } GC_STATIC gc_block_header_t *gc_get_block(void *p) { @@ -914,10 +912,10 @@ HL_API vdynamic *hl_alloc_obj( hl_type *t ) { if( size & (HL_WSIZE-1) ) size += HL_WSIZE - (size & (HL_WSIZE-1)); if( t->kind == HSTRUCT ) { o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? GC_ALLOC_RAW : GC_ALLOC_NOPTR); - o->t = NULL; + o->t = NULL; // TODO: not needed??? (just pass NULL above?) } else { o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); - o->t = t; + o->t = t; // TODO: not needed } for(i=0;inbindings;i++) { hl_runtime_binding *b = rt->bindings + i; @@ -954,7 +952,7 @@ HL_API void hl_gc_set_dump_types( hl_types_dump tdump ) { // GC initialisation ----------------------------------------------- -GC_STATIC void gc_init(void) { +GC_STATIC gc_stats_t *gc_init(void) { gc_mutex_roots = hl_mutex_alloc(false); gc_mutex_pool = hl_mutex_alloc(false); @@ -979,35 +977,47 @@ GC_STATIC void gc_init(void) { gc_mark_stack_next.pos = 0; gc_mark_stack_next.capacity = 0; gc_mark_stack_active = NULL; - gc_stats.live_objects = 0; - gc_stats.live_blocks = 0; - gc_stats.total_memory = 0; - gc_stats.cycles = 0; - gc_stats.total_pages = 0; - // gc_config.memory_limit = 500 * 1024 * 1024; - gc_config.memory_limit = 100 * 1024 * 1024; - gc_config.debug_fatal = true; - gc_config.debug_os = false; - gc_config.debug_page = false; - gc_config.debug_alloc = false; - gc_config.debug_mark = false; - gc_config.debug_sweep = false; - gc_config.debug_other = false; - gc_config.debug_dump = true; - gc_config.dump_file = fopen("gc.dump", "w"); + gc_stats = (gc_stats_t *)calloc(sizeof(gc_stats_t), 1); + gc_stats->live_objects = 0; + gc_stats->live_blocks = 0; + gc_stats->total_memory = 0; + gc_stats->cycles = 0; + gc_stats->total_pages = 0; + gc_config = (gc_config_t *)calloc(sizeof(gc_config_t), 1); + // gc_config->memory_limit = 500 * 1024 * 1024; + gc_config->memory_limit = 100 * 1024 * 1024; + gc_config->debug_fatal = true; + gc_config->debug_os = false; + gc_config->debug_page = false; + gc_config->debug_alloc = false; + gc_config->debug_mark = false; + gc_config->debug_sweep = false; + gc_config->debug_other = false; + gc_config->debug_dump = true; + gc_config->dump_file = fopen("gc.dump", "w"); + return gc_stats; } GC_STATIC void gc_deinit(void) { hl_mutex_free(gc_mutex_roots); hl_mutex_free(gc_mutex_pool); - while (gc_last_allocated_page != NULL) { - gc_free_page_memory(gc_last_allocated_page); + gc_page_header_t *page = gc_last_allocated_page; + gc_page_header_t *next = NULL; + while (page != NULL) { + next = page->next_page; + gc_free_os_memory(page, page->size); + page = next; } - while (gc_last_allocated_huge_page != NULL) { - gc_free_page_memory(gc_last_allocated_huge_page); + page = gc_last_allocated_huge_page; + while (page != NULL) { + next = page->next_page; + gc_free_os_memory(page, page->size); + page = next; } free(gc_page_hash.buckets); free(gc_mark_stack.data); + free(gc_stats); + free(gc_config); } void hl_cache_free(); diff --git a/src/gc.h b/src/gc.h index f829d4180..cfe2008db 100644 --- a/src/gc.h +++ b/src/gc.h @@ -30,6 +30,8 @@ static void gc_free_os_memory(void *ptr, int size); // TODO: check collisions #define GC_HASH(ptr) (((int_val)(ptr) >> 22) ^ (((int_val)(ptr) >> 22) << 5)) +#define GC_STATIC + // GC data structures ----------------------------------------------- typedef enum { @@ -157,15 +159,15 @@ typedef struct { int capacity; } gc_mark_stack_t; -struct { +typedef struct { unsigned long live_objects; unsigned long live_blocks; unsigned long total_memory; unsigned long cycles; int total_pages; -} gc_stats; +} gc_stats_t; -struct { +typedef struct { unsigned long memory_limit; bool debug_fatal; bool debug_os; @@ -176,14 +178,14 @@ struct { bool debug_other; bool debug_dump; FILE *dump_file; -} gc_config; +} gc_config_t; // GC page management ---------------------------------------------- // allocates and initialises a GC page -static gc_page_header_t *gc_alloc_page_memory(void); +GC_STATIC gc_page_header_t *gc_alloc_page_memory(void); // frees a GC page -static void gc_free_page_memory(gc_page_header_t *header); +GC_STATIC void gc_free_page_memory(gc_page_header_t *header); // gets the `block`th block in `page` #define GC_PAGE_BLOCK(page, block) ((gc_block_header_t *)((char *)(page) + (block) * GC_BLOCK_SIZE)) @@ -204,10 +206,10 @@ HL_PRIM void hl_add_root(void **); HL_PRIM void hl_remove_root(void **); // thread-local allocator <-> global GC -static gc_block_header_t *gc_pop_block(void); // size fixed to 4 MiB -static void gc_push_block(gc_block_header_t *); -static void *gc_pop_huge(int size); -static void gc_push_huge(void *); +GC_STATIC gc_block_header_t *gc_pop_block(void); // size fixed to 4 MiB +GC_STATIC void gc_push_block(gc_block_header_t *); +GC_STATIC void *gc_pop_huge(int size); +GC_STATIC void gc_push_huge(void *); // thread setup HL_API int hl_thread_id(void); @@ -221,13 +223,13 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags); HL_API void hl_gc_major(void); // GC internals -static void gc_stop_world(bool); +GC_STATIC void gc_stop_world(bool); -static void gc_mark(void); -static void gc_sweep(void); // or collect, compact, copy, etc +GC_STATIC void gc_mark(void); +GC_STATIC void gc_sweep(void); // or collect, compact, copy, etc // marking and sweeping -static gc_block_header_t *gc_get_block(void *); // NULL if not a GC pointer +GC_STATIC gc_block_header_t *gc_get_block(void *); // NULL if not a GC pointer // GC debug @@ -251,12 +253,12 @@ static gc_block_header_t *gc_get_block(void *); // NULL if not a GC pointer #define GC_FATAL(msg) do { puts(msg); __builtin_trap(); } while (0) /* -static void gc_debug_block(gc_block_header_t *block); -static void gc_debug_general(void); -static void gc_dump(const char *); +GC_STATIC void gc_debug_block(gc_block_header_t *block); +GC_STATIC void gc_debug_general(void); +GC_STATIC void gc_dump(const char *); */ // GC initialisation -static void gc_init(void); -static void gc_deinit(void); +GC_STATIC gc_stats_t *gc_init(void); +GC_STATIC void gc_deinit(void); From 1573f3f2a6799b1416d1b5bed2671c2bc841aa35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Mon, 27 Apr 2020 13:48:43 +0100 Subject: [PATCH 03/13] wip, passes formatter-benchmark, needs heavy cleanup --- Makefile | 1 + other/gctests/Makefile | 7 +- other/gctests/bench/mandelbrot.c | 7 + other/gctests/main.c | 107 ++++- other/gctests/suite.h | 56 ++- other/gctests/utils.h | 2 +- src/gc.c | 782 +++++++++++++++++++++++++++---- src/gc.h | 54 ++- src/hl.h | 21 +- src/std/types.c | 2 +- 10 files changed, 902 insertions(+), 137 deletions(-) diff --git a/Makefile b/Makefile index 09b8b198f..d3111b697 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ INSTALL_DIR ?= $(PREFIX) LIBS=fmt sdl ssl openal ui uv mysql #CFLAGS = -D GC_ENABLE_DEBUG -Wall -O0 -I src -msse2 -mfpmath=sse -std=c11 -I include/pcre -I include/mikktspace -I include/minimp3 -D LIBHL_EXPORTS -D HL_NO_THREADS +#CFLAGS = -Wall -O0 -I src -msse2 -mfpmath=sse -std=c11 -I include/pcre -I include/mikktspace -I include/minimp3 -D LIBHL_EXPORTS -D HL_NO_THREADS CFLAGS = -Wall -O3 -I src -msse2 -mfpmath=sse -std=c11 -I include/pcre -I include/mikktspace -I include/minimp3 -D LIBHL_EXPORTS -D HL_NO_THREADS LFLAGS = -L. -lhl LIBFLAGS = diff --git a/other/gctests/Makefile b/other/gctests/Makefile index 41ba9c80c..e1e7ee516 100644 --- a/other/gctests/Makefile +++ b/other/gctests/Makefile @@ -1,3 +1,6 @@ main: main.c ../../src/gc.h bench/mandelbrot.c - gcc -Wall -Werror -pedantic -Wno-unused-function -Wno-unused-variable -Wno-gnu-zero-variadic-macro-arguments \ - -O3 -std=c11 -o main -I ../../src main.c bench/mandelbrot.c -lhl + gcc -Wall -Werror -pedantic \ + -Wno-unused-function -Wno-unused-variable -Wno-gnu-zero-variadic-macro-arguments \ + -O3 -std=c11 -o main -I ../../src main.c bench/mandelbrot.c ../../libhl.a + +#-O3 -std=c11 -o main -I ../../src main.c bench/mandelbrot.c -lhl diff --git a/other/gctests/bench/mandelbrot.c b/other/gctests/bench/mandelbrot.c index 3ccecd8f4..48ed3f7a5 100644 --- a/other/gctests/bench/mandelbrot.c +++ b/other/gctests/bench/mandelbrot.c @@ -27,6 +27,11 @@ static double hires_timestamp(void) { #endif +static hl_module_context mctx = { + .functions_ptrs = NULL, + .functions_types = NULL +}; + TEST_TYPE(rgb, 3, {0}, { int r; int g; @@ -71,6 +76,8 @@ static hlt_complex_t *complexSquare(hlt_complex_t *val) { } BEGIN_BENCH_CASE(mandelbrot) { + hl_alloc_init(&mctx.alloc); + varray *palette = hl_alloc_array(&hlt_rgb, MB_MAX_ITERATIONS + 1); hl_add_root((void *)&palette); for (int i = 0; i < MB_MAX_ITERATIONS + 1; i++) { diff --git a/other/gctests/main.c b/other/gctests/main.c index 4c4f76673..d34fc665e 100644 --- a/other/gctests/main.c +++ b/other/gctests/main.c @@ -7,10 +7,15 @@ // types -TEST_TYPE(one_ptr, 1, {1}, { +static hl_module_context mctx = { + .functions_ptrs = NULL, + .functions_types = NULL +}; + +TEST_TYPE(one_ptr, 1, {2}, { void *ptr; }); -TEST_TYPE(some_ptr, 10, {1 | 4 | 64 | 128 | 512}, { +TEST_TYPE(some_ptr, 10, {2 | 8 | 128 | 256 | 1024}, { void *ptr_0; void *data_1; void *ptr_2; @@ -22,13 +27,18 @@ TEST_TYPE(some_ptr, 10, {1 | 4 | 64 | 128 | 512}, { void *data_8; void *ptr_9; }); -TEST_TYPE(medium_one_ptr, 256, {1}, { +TEST_TYPE(medium_one_ptr, 256, {2}, { void *ptr; void *data[255]; }); TEST_TYPE(big_no_ptr, 10000, {0}, { int _dummy; }); +TEST_TYPE(bin_tree, 3, {6}, { + void *left; + void *right; + int value; +}); // test cases @@ -36,7 +46,7 @@ BEGIN_TEST_CASE(sanity) { ASSERT(sizeof(hl_type *) == 8); ASSERT(sizeof(gc_page_header_t) == 32); ASSERT(sizeof(gc_metadata_t) == 1); - ASSERT(sizeof(gc_metadata_ext_t) == 2); + // ASSERT(sizeof(gc_metadata_ext_t) == 2); ASSERT(sizeof(gc_block_header_t) == GC_BLOCK_SIZE); ASSERT(offsetof(gc_block_header_t, lines) == 64 * GC_LINE_SIZE); @@ -68,6 +78,19 @@ BEGIN_TEST_CASE(sanity) { ASSERT(GC_METADATA(0x686178650B4D2018ul) == (gc_metadata_t *)0x686178650B4D0203ul); ASSERT(GC_METADATA(0x686178650B4D2128ul) == (gc_metadata_t *)0x686178650B4D0225ul); ASSERT(GC_METADATA(0x686178650B4DFFF0ul) == (gc_metadata_t *)0x686178650B4D1DFEul); + + char cs[] = "\x00\x01\x02\x03\x04\x05"; + ASSERT(*(int *)(&cs[0]) == 0x03020100); + ASSERT(*(int *)(&cs[1]) == 0x04030201); +} END_TEST_CASE + +BEGIN_TEST_CASE(thread_mark) { + hlt_one_ptr_t *obj = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); + + hl_gc_major(); + ASSERT(gc_stats->live_objects == 1); + + printf("%p\n", obj); } END_TEST_CASE // a cycle of one object @@ -93,9 +116,9 @@ BEGIN_TEST_CASE(tiny_cycle) { hl_gc_major(); ASSERT(gc_stats->live_objects == 3); - ASSERT(gc_stats->total_pages == 1); + // ASSERT(gc_stats->total_pages == 1); hl_gc_major(); - ASSERT(gc_stats->live_objects == 3); + ASSERT_EQlu(gc_stats->live_objects, 3); } END_TEST_CASE // a cycle spanning more than one block @@ -191,7 +214,8 @@ BEGIN_TEST_CASE(medium_object_recycle) { } hl_gc_major(); - ASSERT(gc_stats->live_objects == OBJ_COUNT - gap); + printf("got: %lu, expected at least: %d\n", gc_stats->live_objects, OBJ_COUNT - gap); + ASSERT(gc_stats->live_objects >= OBJ_COUNT - gap); objs[1338] = (hlt_one_ptr_t *)hl_alloc_obj(&hlt_one_ptr); hlt_medium_one_ptr_t *med = (hlt_medium_one_ptr_t *)hl_alloc_obj(&hlt_medium_one_ptr); @@ -199,7 +223,8 @@ BEGIN_TEST_CASE(medium_object_recycle) { objs[1338]->ptr = med; med->ptr = objs[1338 + gap]; hl_gc_major(); - ASSERT(gc_stats->live_objects == OBJ_COUNT - gap + 2); + //ASSERT(gc_stats->live_objects == OBJ_COUNT - gap + 2); + printf("%p %p\n", gc_get_block(objs[0]), gc_get_block(objs[1338])); ASSERT(gc_get_block(objs[0]) == gc_get_block(objs[1338])); ASSERT(gc_get_block(objs[0]) == gc_get_block(med)); } END_TEST_CASE @@ -212,7 +237,7 @@ BEGIN_TEST_CASE(big_object) { hl_gc_major(); ASSERT(gc_stats->live_objects == 2); - ASSERT(gc_stats->total_pages == 2); + // ASSERT(gc_stats->total_pages_normal == 1); printf("%p %p\n", obj, &obj); // force stack variable for obj printf("%p %p\n", big, &big); // force stack variable for big @@ -240,12 +265,12 @@ BEGIN_TEST_CASE(simple_array) { } hl_gc_major(); - ASSERT(gc_stats->live_objects == 41); + ASSERT_EQlu(gc_stats->live_objects, 41); } END_TEST_CASE BEGIN_TEST_CASE(big_array) { varray *arr = hl_alloc_array(&hlt_one_ptr, 15000); - //hl_add_root((void *)&arr); + printf("array: %p\n", arr); hl_gc_major(); ASSERT(gc_stats->live_objects == 1); @@ -259,9 +284,44 @@ BEGIN_TEST_CASE(big_array) { hl_gc_major(); ASSERT(gc_stats->live_objects == 3); - printf("%p %p\n", arr, &arr); // force stack variable for arr + printf("%p\n", arr); //, &arr); // force stack variable for arr } END_TEST_CASE +void helper_test_many_trees_alloc(hlt_bin_tree_t *root, int depth, int ctr) { + root->value = ctr; + if (depth > 0) { + hlt_bin_tree_t *left = (hlt_bin_tree_t *)hl_alloc_obj(&hlt_bin_tree); + hlt_bin_tree_t *right = (hlt_bin_tree_t *)hl_alloc_obj(&hlt_bin_tree); + helper_test_many_trees_alloc(left, depth - 1, ctr * 2 + 1); + helper_test_many_trees_alloc(right, depth - 1, ctr * 2 + 2); + root->left = left; + root->right = right; + } +} + +BEGIN_TEST_CASE(many_trees) { + int depth = 21; + hlt_bin_tree_t *root = (hlt_bin_tree_t *)hl_alloc_obj(&hlt_bin_tree); + helper_test_many_trees_alloc(root, depth, 0); + hl_gc_major(); + ASSERT(gc_stats->live_objects == (1 << (depth + 1)) - 1); +} END_TEST_CASE + +void helper_test_finalizer(hlt_some_ptr_t *data) { + *((int *)data->ptr_2) = 1; +} + +BEGIN_TEST_CASE_C(finalizer, int finalizer_called = 0;) { + hlt_some_ptr_t *obj = hl_gc_alloc_finalizer(sizeof(hlt_some_ptr_t)); + obj->t = (void *)helper_test_finalizer; + obj->ptr_2 = &finalizer_called; + obj = NULL; + hl_gc_major(); + ASSERT(gc_stats->live_objects == 0); +} END_TEST_CASE_C({ + ASSERT(finalizer_called == 1); +}) + // benchmark cases void bench_mandelbrot(int); @@ -269,23 +329,28 @@ void bench_mandelbrot(int); // test runner int main(int argc, char **argv) { + hl_alloc_init(&mctx.alloc); int assertions = 0; int assertions_successful = 0; - test_sanity(&assertions, &assertions_successful); - test_self_cycle(&assertions, &assertions_successful); - test_tiny_cycle(&assertions, &assertions_successful); - test_block_cycle(&assertions, &assertions_successful); - test_complex_cycle(&assertions, &assertions_successful); - test_medium_object_recycle(&assertions, &assertions_successful); - test_big_object(&assertions, &assertions_successful); - test_simple_array(&assertions, &assertions_successful); - test_big_array(&assertions, &assertions_successful); + RUN_TEST(sanity); + RUN_TEST(thread_mark); + RUN_TEST(self_cycle); + RUN_TEST(tiny_cycle); + RUN_TEST(block_cycle); + RUN_TEST(complex_cycle); + //RUN_TEST(medium_object_recycle); + RUN_TEST(big_object); + RUN_TEST(simple_array); + RUN_TEST(big_array); + //RUN_TEST(many_trees); + RUN_TEST(finalizer); puts("---"); puts("TOTAL:"); printf(" %d / %d checks passed\n", assertions_successful, assertions); if (assertions_successful != assertions) return 1; + return 0; bench_mandelbrot(1); } diff --git a/other/gctests/suite.h b/other/gctests/suite.h index ff45f99dc..b8bc0d13b 100644 --- a/other/gctests/suite.h +++ b/other/gctests/suite.h @@ -1,6 +1,6 @@ #pragma once -#define BEGIN_TEST_CASE(name) \ +#define BEGIN_TEST_CASE_C(name, setup) \ void test_ ## name (int *_total_assertions, int *_total_assertions_successful) { \ puts("TEST: " # name); \ gc_stats_t *gc_stats = gc_init(); \ @@ -8,13 +8,22 @@ const char *_test_name = #name; \ int _assertions = 0; \ int _assertions_successful = 0; \ + setup \ do -#define END_TEST_CASE \ +#define END_TEST_CASE_C(cleanup) \ while (0); \ - printf("%d / %d checks passed\n", _assertions_successful, _assertions); \ hl_unregister_thread(); \ gc_deinit(); \ + cleanup \ + printf("%d / %d checks passed\n", _assertions_successful, _assertions); \ } +#define BEGIN_TEST_CASE(name) BEGIN_TEST_CASE_C(name, ) +#define END_TEST_CASE END_TEST_CASE_C() +#define RUN_TEST(name) \ + do { \ + void (*volatile f)(int *, int *) = test_ ## name; \ + f(&assertions, &assertions_successful); \ + } while (0) #define BEGIN_BENCH_CASE(name) \ void bench_ ## name (int _bench_count) { \ @@ -45,3 +54,44 @@ gc_deinit(); \ return; \ } + +#define ASSERT_EQlu(lhs, rhs) \ + _assertions++; \ + (*_total_assertions)++; \ + do { \ + unsigned long _lhs = (lhs); \ + unsigned long _rhs = (rhs); \ + if (lhs == rhs) { \ + _assertions_successful++; \ + (*_total_assertions_successful)++; \ + printf("\x1B[38;5;28m[.] assertion passed in %s (line %d): " #lhs " == " #rhs "\n\x1B[0m", _test_name, __LINE__); \ + } else { \ + printf("\x1B[38;5;160m[!] assertion failed in %s (line %d): " #lhs " == " #rhs "\n\x1B[0m", _test_name, __LINE__); \ + printf("\x1B[38;5;160m[!] (" #lhs " == %lu)\n\x1B[0m", _lhs); \ + printf("\x1B[38;5;160m[!] (" #rhs " == %lu)\n\x1B[0m", _rhs); \ + printf("%d / %d checks passed\n", _assertions_successful, _assertions); \ + hl_unregister_thread(); \ + gc_deinit(); \ + return; \ + } \ + } while (0) +#define ASSERT_EQd(lhs, rhs) \ + _assertions++; \ + (*_total_assertions)++; \ + do { \ + int _lhs = (lhs); \ + int _rhs = (rhs); \ + if (lhs == rhs) { \ + _assertions_successful++; \ + (*_total_assertions_successful)++; \ + printf("\x1B[38;5;28m[.] assertion passed in %s (line %d): " #lhs " == " #rhs "\n\x1B[0m", _test_name, __LINE__); \ + } else { \ + printf("\x1B[38;5;160m[!] assertion failed in %s (line %d): " #lhs " == " #rhs "\n\x1B[0m", _test_name, __LINE__); \ + printf("\x1B[38;5;160m[!] (" #lhs " == %d)\n\x1B[0m", _lhs); \ + printf("\x1B[38;5;160m[!] (" #rhs " == %d)\n\x1B[0m", _rhs); \ + printf("%d / %d checks passed\n", _assertions_successful, _assertions); \ + hl_unregister_thread(); \ + gc_deinit(); \ + return; \ + } \ + } while (0) diff --git a/other/gctests/utils.h b/other/gctests/utils.h index 3078e42cf..301b110fe 100644 --- a/other/gctests/utils.h +++ b/other/gctests/utils.h @@ -32,7 +32,7 @@ .nmethods = 0, .nbindings = 0, .hasPtr = true, .methods = NULL, \ .fields_indexes = NULL, .bindings = NULL, .parent = NULL, .toStringFun = NULL, \ .compareFun = NULL, .castFun = NULL, .getFieldFun = NULL, .nlookup = 0, .lookup = NULL }; \ - hl_type_obj hlto_ ## name = { .rt = &hltr_ ## name }; \ + hl_type_obj hlto_ ## name = { .m = &mctx, .rt = &hltr_ ## name }; \ unsigned int hltm_ ## name[(words + 31) / 32] = bits; \ hl_type hlt_ ## name = { HOBJ, .obj = &hlto_ ## name, .mark_bits = hltm_ ## name }; \ typedef struct { \ diff --git a/src/gc.c b/src/gc.c index 70ff318c6..6703d71b6 100644 --- a/src/gc.c +++ b/src/gc.c @@ -31,21 +31,42 @@ // bit access #define GET_BIT(bmp, bit) ((bmp)[(bit) >> 3] & (1 << ((bit) & 7))) +#define GET_BIT32(bmp, bit) ((bmp)[(bit) >> 5] & (1 << ((bit) & 31))) #define SET_BIT0(bmp, bit) (bmp)[(bit) >> 3] &= 0xFF ^ (1 << ((bit) & 7)); #define SET_BIT1(bmp, bit) (bmp)[(bit) >> 3] |= 1 << ((bit) & 7); #define SET_BIT(bmp, bit, val) (bmp)[(bit) >> 3] = ((bmp)[(bit) >> 3] & (0xFF ^ (1 << ((bit) & 7)))) | ((val) << ((bit) & 7)); +// singly-linked lists +#define SLL_POP_P(obj, list, next_field) do { \ + GC_ASSERT(list != NULL); \ + (obj) = (list); \ + (list) = (obj)->next_field; \ + (obj)->next_field = NULL; \ + } while (0) +#define SLL_POP(obj, list) SLL_POP_P(obj, list, next) + +#define SLL_PUSH_P(obj, list, next_field) do { \ + (obj)->next_field = list; \ + list = (obj); \ + } while (0) +#define SLL_PUSH(obj, list) SLL_PUSH_P(obj, list, next) + // doubly-linked lists #define DLL_UNLINK(obj) do { \ + GC_ASSERT((obj) != NULL); \ + GC_ASSERT_M((obj)->next != NULL, "%p", (obj)); \ + GC_ASSERT_M((obj)->prev != NULL, "%p", (obj)); \ (obj)->next->prev = (obj)->prev; \ - (obj)->prev->next = (obj)->next; \ - (obj)->next = NULL; \ + /* printf("%p->n := %p\n", (obj)->prev, (obj)->next); */ (obj)->prev->next = (obj)->next; \ + /* printf("%p->n := NULL\n", (obj)->next); */ (obj)->next = NULL; \ (obj)->prev = NULL; \ } while (0) #define DLL_INSERT(obj, head) do { \ - (obj)->next = (head)->next; \ + GC_ASSERT((obj) != NULL); \ + GC_ASSERT((head) != NULL); \ + /* printf("%p->n := %p\n", (obj), (head)->next); */ (obj)->next = (head)->next; \ (obj)->prev = (head); \ - (head)->next = (obj); \ + /* printf("%p->n := %p\n", (head), (obj)); */ (head)->next = (obj); \ (obj)->next->prev = (obj); \ } while (0) @@ -54,9 +75,25 @@ static gc_stats_t *gc_stats; static gc_config_t *gc_config; +static bool gc_blocked; + // OS page management ---------------------------------------------- static void *base_addr; +static int page_counter = 0; +static int huge_counter = 0; +static gc_page_header_t *tracked_page_addr[500] = {0}; +static gc_page_header_t *tracked_huge_addr[500] = {0}; +// static bool do_log = false; + +#define ID_TRACK(page_num, low) \ + (gc_object_t *)((int_val)tracked_page_addr[page_num] | ((int_val)low & 0x3FFFFF)) + +static FILE *dump_file; +/*void dump_live(void) { + fwrite(&(gc_stats->live_memory), 8, 1, dump_file); +}*/ +#define dump_live(...) // allocates a page-aligned region of memory from the OS GC_STATIC void *gc_alloc_os_memory(int size) { @@ -71,7 +108,6 @@ GC_STATIC void *gc_alloc_os_memory(int size) { GC_DEBUG_DUMP0("gc_alloc_os_memory.fail.other"); GC_FATAL("failed to allocate page"); } - gc_stats->total_memory += size; if (((int_val)ptr) & (GC_PAGE_SIZE - 1)) { // re-align munmap(ptr,size); @@ -92,7 +128,9 @@ GC_STATIC void *gc_alloc_os_memory(int size) { munmap(tmp, tmp_size); return ptr; } + gc_stats->total_memory += size; base_addr = (char *)ptr + size; +//printf("PAGE ALLOC %d: %p (%d bytes)\n", page_counter++, ptr, size); GC_DEBUG_DUMP2("gc_alloc_os_memory.success", ptr, size); GC_DEBUG(os, "allocated OS page: %p", ptr); return ptr; @@ -124,7 +162,7 @@ GC_STATIC void gc_add_page(gc_page_header_t *header) { if ((char *)header < (char *)gc_min_allocated) { gc_min_allocated = (char *)header; } - if ((char *)header + GC_PAGE_SIZE > (char *)gc_max_allocated) { + if ((char *)header + GC_PAGE_SIZE > (char *)gc_max_allocated) { // TODO: this over-estimates max alloc gc_max_allocated = (char *)header + GC_PAGE_SIZE; } @@ -166,18 +204,27 @@ GC_STATIC gc_page_header_t *gc_alloc_page_normal(void) { header->next_page = gc_last_allocated_page; gc_last_allocated_page = header; + gc_stats->total_pages_normal++; gc_add_page(header); +if (page_counter < 500) + tracked_page_addr[page_counter++] = header; + gc_block_header_t *blocks = GC_PAGE_BLOCK(header, 0); for (int i = 0; i < GC_BLOCKS_PER_PAGE; i++) { blocks[i].kind = GC_BLOCK_FREE; blocks[i].next = &blocks[i + 1]; + blocks[i].debug_objects = (char *)calloc(1, 896); + if (blocks[i].debug_objects == NULL) + GC_FATAL("cannot alloc debug bitmap?"); } hl_mutex_acquire(gc_mutex_pool); blocks[GC_BLOCKS_PER_PAGE - 1].next = gc_pool; gc_pool = &blocks[0]; gc_pool_count += GC_BLOCKS_PER_PAGE; hl_mutex_release(gc_mutex_pool); +//printf("PAGE %p\n", header); +gc_debug_verify_pool("page"); GC_DEBUG_DUMP1("gc_alloc_page_normal.success", header); return header; @@ -185,6 +232,9 @@ GC_STATIC gc_page_header_t *gc_alloc_page_normal(void) { // allocates a huge page (containing a single object) GC_STATIC gc_page_header_t *gc_alloc_page_huge(int size) { +//printf("HUGE before\n"); +gc_debug_verify_pool("huge before"); + gc_page_header_t *header = (gc_page_header_t *)gc_alloc_os_memory(sizeof(gc_page_header_t) + size); if (header == NULL) { GC_FATAL("OOM: huge page"); @@ -197,6 +247,12 @@ GC_STATIC gc_page_header_t *gc_alloc_page_huge(int size) { gc_add_page(header); +if (huge_counter < 500) + tracked_huge_addr[huge_counter++] = header; + +//printf("HUGE %p\n", header); +gc_debug_verify_pool("huge after"); + GC_DEBUG_DUMP2("gc_alloc_page_huge.success", header, size); return header; } @@ -228,6 +284,13 @@ GC_STATIC void gc_free_page_memory(gc_page_header_t *header) { GC_DEBUG_DUMP1("gc_free_page_memory.success", header); } +GC_STATIC void gc_grow_heap(int count) { +//printf("GROWTH %d\n", count); + for (int i = 0; i < count; i++) { + gc_alloc_page_normal(); + } +} + // roots ----------------------------------------------------------- static hl_mutex *gc_mutex_roots; @@ -294,11 +357,6 @@ HL_API void hl_register_thread(void *stack_top) { t->stack_top = stack_top; t->flags = HL_TRACK_MASK << HL_TREAD_TRACK_SHIFT; - gc_block_header_t *block = gc_pop_block(); - t->lines_block = block; - t->lines_start = &block->lines[0]; - t->lines_limit = &block->lines[GC_LINES_PER_BLOCK]; - gc_block_dummy_t *sentinels = (gc_block_dummy_t *)calloc(sizeof(gc_block_dummy_t), 4); sentinels[0].next = (gc_block_header_t *)&sentinels[1]; sentinels[1].prev = (gc_block_header_t *)&sentinels[0]; @@ -313,6 +371,11 @@ HL_API void hl_register_thread(void *stack_top) { hl_add_root((void **)&t->exc_value); hl_add_root((void **)&t->exc_handler); + gc_block_header_t *block = gc_pop_block(); + t->lines_block = block; + t->lines_start = &block->lines[0]; + t->lines_limit = &block->lines[GC_LINES_PER_BLOCK]; + /* // TODO: thread list gc_global_lock(true); @@ -386,7 +449,9 @@ GC_STATIC void gc_stop_world( bool b ) { // acquires a block for the current thread // the block may come from the thread's recycle list, or from the global // free block pool (which incurs a locking penalty) +// TODO: recycle list is only touched in alloc? GC_STATIC gc_block_header_t *gc_pop_block(void) { +gc_debug_verify_pool("pop before"); hl_mutex_acquire(gc_mutex_pool); // TODO: mutexes need to be re-entrant, or lock later if (UNLIKELY(gc_pool_count == 0)) { // if (gc_stats->total_memory * 1.2 >= gc_config->memory_limit) { @@ -400,27 +465,35 @@ GC_STATIC gc_block_header_t *gc_pop_block(void) { GC_FATAL("OOM: page memory"); } } - gc_block_header_t *block = gc_pool; - gc_pool = block->next; + gc_block_header_t *block; + SLL_POP(block, gc_pool); gc_page_header_t *page = GC_BLOCK_PAGE(block); // TODO: mutex per page? page->free_blocks--; SET_BIT1(page->block_bmp, GC_BLOCK_ID(block)); gc_pool_count--; +/*printf("POP pool count: %d block: %p page: %p sen: %p %p %p %p pool: %p\n", + gc_pool_count, block, page, + ¤t_thread->sentinels[0], + ¤t_thread->sentinels[1], + ¤t_thread->sentinels[2], + ¤t_thread->sentinels[3], + gc_pool); fflush(stdout);*/ hl_mutex_release(gc_mutex_pool); - block->kind = GC_BLOCK_NEW; + block->kind = GC_BLOCK_BRAND_NEW; block->owner_thread = hl_thread_id(); GC_DEBUG(alloc, "got block %p", block); GC_DEBUG_DUMP1("gc_pop_block.success", block); gc_stats->live_blocks++; +gc_debug_verify_pool("pop after"); + GC_ASSERT(block != NULL); return block; } // returns a thread-owned block to the global free pool GC_STATIC void gc_push_block(gc_block_header_t *block) { - if (block->kind == GC_BLOCK_FULL || block->kind == GC_BLOCK_RECYCLED) { - DLL_UNLINK(block); - } // TODO: condition should always be true, until zombie blocks are introduced ? + GC_ASSERT(block != NULL); + GC_ASSERT(block->kind == GC_BLOCK_FULL); block->kind = GC_BLOCK_FREE; // TODO: zombie ? block->owner_thread = -1; memset(block->line_marks, 0, GC_LINES_PER_BLOCK); @@ -430,11 +503,12 @@ GC_STATIC void gc_push_block(gc_block_header_t *block) { page->free_blocks++; SET_BIT0(page->block_bmp, GC_BLOCK_ID(block)); // TODO: manage disorder in GC pool (prefer address order) - block->next = gc_pool; - gc_pool = block; + SLL_PUSH(block, gc_pool); gc_pool_count++; +//printf("PUSH pool count: %d block: %p (%d) page: %p sen: %p pool: %p\n", gc_pool_count, block, block->kind, page, current_thread ? current_thread->sentinels : NULL, gc_pool); fflush(stdout); gc_stats->live_blocks--; hl_mutex_release(gc_mutex_pool); +gc_debug_verify_pool("push"); GC_DEBUG_DUMP1("gc_push_block.success", block); } @@ -451,6 +525,8 @@ GC_STATIC void *gc_pop_huge(int size) { // frees a huge object GC_STATIC void gc_push_huge(void *huge) { + GC_ASSERT(huge != NULL); + gc_page_header_t *header = GC_BLOCK_PAGE(huge); GC_DEBUG(alloc, "freed huge %p", huge); GC_DEBUG_DUMP1("gc_push_huge.success", header); @@ -464,7 +540,13 @@ GC_STATIC void gc_push_huge(void *huge) { // the object // this also initialises the object header GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) { + GC_ASSERT(words > 0); + GC_ASSERT(size > 0); + +// printf("pre alloc type: %p size: %d words: %d flags: %d start: %p\n", t, size, words, flags, current_thread->lines_start); + gc_object_t *ret = (gc_object_t *)current_thread->lines_start; + gc_block_header_t *block = GC_LINE_BLOCK(ret); gc_metadata_t *meta = GC_METADATA(ret); // clear stale data @@ -473,11 +555,12 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) ret->t = t; meta->flags = 0; meta->marked = !gc_mark_polarity; - meta->words = words & 15; - if (flags & GC_ALLOC_DYNAMIC) { - meta->dynamic_mark = 1; + int sub_words = words - 1; + meta->words = sub_words & 15; + if (flags & GC_ALLOC_FLAG_RAW) { + meta->raw = 1; } - if (flags & GC_ALLOC_NOPTR) { + if (flags & GC_ALLOC_FLAG_NOPTR) { meta->no_ptr = 1; } // int line_id = GC_LINE_ID(ret); @@ -488,20 +571,54 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) // GC_DEBUG("allocated small %p in block %p (%d bytes, %d words)", ret, block, size, words); } else { meta->medium_sized = 1; - gc_metadata_ext_t *meta_ext = (gc_metadata_ext_t *)meta; - meta_ext->words_ext = words >> 4; + GC_METADATA_EXT(ret) = sub_words >> 4; // int last_id = GC_LINE_ID((void *)ret + size); // ret->line_span = (last_id - line_id) + 1; // GC_DEBUG("allocated medium %p in block %p (%d bytes, %d lines, %d words)", ret, block, size, ret->line_span, words); } + if (flags == GC_ALLOC_FINALIZER) { + // TODO: also in a finaliser-specific alloc function (after alloc_gen call) + block->line_finalize[GC_LINE_ID(ret)] = 1; + } + GC_DEBUG_DUMP3("gc_alloc_bump.success", ret, size, flags); + gc_stats->live_memory += size; + gc_stats->live_memory_normal += size; + dump_live(); + + int obj_id = ((int_val)ret - (int_val)&block->lines[0]) / 8; +// if (GET_BIT(block->debug_objects, obj_id)) { +//printf("%p (block %p, size %d, obj %d)\n", ret, block, size, obj_id); +//gc_debug_block(block); +// GC_FATAL("overriding existing object"); +// } + SET_BIT1(block->debug_objects, obj_id); + +// if (do_log) +// printf("postalloc: %p (%d bytes, %d flags)\n", ret, size, flags); + +//gc_page_header_t *page = GC_BLOCK_PAGE(block); +//if (ret == ID_TRACK(9, 0x2b5c80)) { +//if (ret >= ID_TRACK(12, 0x188900) && ret < ID_TRACK(12, 0x188980)) { +// gc_debug_block(block); +// printf("tracked alloc: %p (%d bytes, %d flags)\n", ret, size, flags); +// gc_debug_obj(ret); +// gc_debug_interactive(); +//} + return ret; } // finds the next line gap in the active block of the current thread // returns true if a gap was found GC_STATIC bool gc_find_gap(void) { + GC_ASSERT( + current_thread->lines_block->kind == GC_BLOCK_BRAND_NEW + || current_thread->lines_block->kind == GC_BLOCK_NEW + || current_thread->lines_block->kind == GC_BLOCK_RECYCLED + ); + if (GC_LINE_BLOCK(current_thread->lines_start) != current_thread->lines_block) return false; // TODO: optimise @@ -534,14 +651,33 @@ GC_STATIC bool gc_find_gap(void) { // generic allocation function HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { + GC_ASSERT(!gc_blocked); + + if (size < sizeof(void *)) { + size = sizeof(void *); + } + // align to words int words = (size + (sizeof(void *) - 1)) / sizeof(void *); + + if (flags == GC_ALLOC_FINALIZER) { + // TODO: separate function to do this before alloc_gen + // (to reduce hot path for non-finaliser objects) + + // align to cache lines + words = (((size + 127) / 128) * 128) / sizeof(void *); + current_thread->lines_start = (int_val)((char *)current_thread->lines_start + 127) & 0xFFFFFFFFFFFFFF80; + } + size = words * sizeof(void *); if (LIKELY((char *)current_thread->lines_start + size <= (char *)current_thread->lines_limit)) { return gc_alloc_bump(t, size, words, flags); } + // align start to next cache line (for gc_find_gap) + current_thread->lines_start = (int_val)((char *)current_thread->lines_start + 127) & 0xFFFFFFFFFFFFFF80; + // TODO: list operations should not be interrupted by the GC, mutex per list if (LIKELY(size <= GC_LINE_SIZE)) { @@ -553,70 +689,92 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { // add current block to full blocks current_thread->lines_block->kind = GC_BLOCK_FULL; DLL_INSERT(current_thread->lines_block, current_thread->full_blocks); + gc_debug_verify_pool("full insert small"); // no gap found, get next recyclable block if (current_thread->recyclable_blocks->next->next != NULL) { current_thread->lines_block = current_thread->recyclable_blocks->next; current_thread->lines_start = ¤t_thread->lines_block->lines[0]; DLL_UNLINK(current_thread->lines_block); - current_thread->lines_block->kind = GC_BLOCK_NEW; // TODO: maybe a separate kind + current_thread->lines_block->kind = GC_BLOCK_NEW; if (gc_find_gap()) { return gc_alloc_bump(t, size, words, flags); } + gc_debug_block(current_thread->lines_block); GC_FATAL("unreachable"); } // if none, get fresh block current_thread->lines_block = gc_pop_block(); - if (current_thread->lines_block == NULL) { - hl_gc_major(); - current_thread->lines_block = gc_pop_block(); - if (current_thread->lines_block == NULL) { - GC_FATAL("OOM: alloc"); - } - } current_thread->lines_start = ¤t_thread->lines_block->lines[0]; current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; return gc_alloc_bump(t, size, words, flags); } else if (size <= GC_MEDIUM_SIZE) { current_thread->lines_block->kind = GC_BLOCK_FULL; DLL_INSERT(current_thread->lines_block, current_thread->full_blocks); + gc_debug_verify_pool("full insert medium"); // get empty block to avoid expensive search current_thread->lines_block = gc_pop_block(); - if (current_thread->lines_block == NULL) { - hl_gc_major(); - current_thread->lines_block = gc_pop_block(); - if (current_thread->lines_block == NULL) { - GC_FATAL("OOM: alloc"); - } - } current_thread->lines_start = ¤t_thread->lines_block->lines[0]; current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; return gc_alloc_bump(t, size, words, flags); } else { + // TODO: separate path for huge objects +if (flags == GC_ALLOC_FINALIZER) { + GC_FATAL("cannot alloc huge finalizer"); +} gc_object_t *obj = gc_pop_huge(size); obj->t = t; gc_metadata_t *meta = GC_METADATA(obj); meta->flags = 0; meta->marked = !gc_mark_polarity; - if (flags & GC_ALLOC_DYNAMIC) { - meta->dynamic_mark = 1; + if (flags & GC_ALLOC_FLAG_RAW) { + meta->raw = 1; } - if (flags & GC_ALLOC_NOPTR) { + if (flags & GC_ALLOC_FLAG_NOPTR) { meta->no_ptr = 1; } + gc_stats->live_memory += size; + dump_live(); return obj; } } -// triggers a major GC collection +GC_STATIC void **get_stack_bottom(void) { + void *x; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreturn-stack-address" + return &x; +#pragma clang diagnostic pop +} + +// spills registers and triggers a major GC collection HL_API void hl_gc_major(void) { - gc_stop_world(true); + puts("major cycle..."); + //gc_stop_world(true); + //* + jmp_buf env; + setjmp(env); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wincompatible-pointer-types" + void *(*volatile f)(void) = get_stack_bottom; +#pragma clang diagnostic pop + current_thread->stack_cur = f(); + /* + printf("SPILLED REGISTERS:\n"); + void **a = &env; + void **b = (void **)((char *)a + sizeof(jmp_buf)); + for (; a < b; a++) { + printf("reg: %p\n", *a); + } + printf("marking thread from %p to %p\n", current_thread->stack_cur, current_thread->stack_top); + */ + //*/ GC_DEBUG_DUMP0("hl_gc_major.entry"); gc_mark(); gc_sweep(); - gc_stop_world(false); + // gc_stop_world(false); GC_DEBUG_DUMP0("hl_gc_major.success"); } @@ -658,6 +816,7 @@ GC_STATIC void gc_push(void *p, void *ref) { gc_mark_total++; gc_mark_stack_active->data[gc_mark_stack_active->pos].object = p; gc_mark_stack_active->data[gc_mark_stack_active->pos++].reference = ref; +//gc_page_header_t *page = GC_BLOCK_PAGE(p); GC_DEBUG(mark, "pushed %p -> %p to %p", ref, p, gc_mark_stack_active->data); GC_DEBUG_DUMP2("gc_push.success", p, ref); } @@ -667,7 +826,7 @@ GC_STATIC void gc_push(void *p, void *ref) { // get the next pointer (and its incoming reference) from the mark stack GC_STATIC gc_object_t *gc_pop(void) { void *p = gc_mark_stack.data[--gc_mark_stack.pos].object; - void *ref = gc_mark_stack.data[gc_mark_stack.pos].reference; + // void *ref = gc_mark_stack.data[gc_mark_stack.pos].reference; // GC_DEBUG("[M] popped %p from %p", p, gc_mark_stack.data); if (gc_mark_stack.pos <= 0 && gc_mark_stack_active != &gc_mark_stack) { GC_DEBUG(mark, "mark stack shrunk"); @@ -685,17 +844,21 @@ GC_STATIC gc_object_t *gc_pop(void) { HL_PRIM vbyte* hl_type_name( hl_type *t ); +static unsigned long debug_scan_ctr = 0; + // marks live objects in the heap GC_STATIC void gc_mark(void) { GC_DEBUG(mark, "start"); GC_DEBUG(mark, "polarity: %d", gc_mark_polarity); - // TODO: only if debug // reset stats gc_stats->live_objects = 0; + gc_stats->live_memory = 0; + gc_stats->live_memory_normal = 0; // reset line marks // TODO: is there a better place to do this? + // TODO: use a rolling 8-bit polarity, only reset every 256 cycles for (gc_page_header_t *page = gc_last_allocated_page; page != NULL; page = page->next_page) { for (int block_id = 0; block_id < GC_BLOCKS_PER_PAGE; block_id++) { if (GET_BIT(page->block_bmp, block_id)) { @@ -728,13 +891,20 @@ GC_STATIC void gc_mark(void) { void **cur = t->stack_cur; void **top = t->stack_top; while (cur < top) { - gc_push(*cur++, NULL); - } +//printf("stack: %p: %p\n", cur, *cur); + if (*cur == ID_TRACK(9, 0x34ab10)) { + gc_debug_obj_id(*cur); + printf("tracked push %p -> %p\n", cur, *cur); + } + gc_push(*cur, NULL); + cur++; + }/* cur = (void **)(&(t->gc_regs)); top = (void **)(((char *)&(t->gc_regs)) + sizeof(jmp_buf)); while (cur < top) { + printf("reg: %p\n", *cur); gc_push(*cur++, NULL); - } + }*/ } GC_DEBUG_DUMP0("gc_mark.propagate"); @@ -743,54 +913,96 @@ GC_STATIC void gc_mark(void) { while (gc_mark_total-- > 0) { gc_stats->live_objects++; gc_object_t *p = gc_pop(); +//if (debug_scan_ctr == 84) +// gc_debug_obj(p); +//printf("live object: %p %lu\n", p, debug_scan_ctr++); if (p == NULL) GC_FATAL("popped null"); gc_metadata_t *meta = GC_METADATA(p); + gc_block_header_t *block = GC_LINE_BLOCK(p); + +//if (p == ID_TRACK(9, 0x1af08)) +// puts("popped lock"); // mark line(s) int line_id = GC_LINE_ID(p); + int obj_id = ((int_val)p - (int_val)&block->lines[0]) / 8; int words = meta->words; - gc_block_header_t *block = GC_LINE_BLOCK(p); - if (meta->medium_sized) { - gc_metadata_ext_t *meta_ext = (gc_metadata_ext_t *)meta; - words |= (meta_ext->words_ext << 4); + + /* + if (words == 0) { + if (GC_BLOCK_PAGE(p)->kind != GC_PAGE_HUGE && !GET_BIT(block->debug_objects, obj_id)) { + puts("non-ex!"); + } + gc_debug_obj(p); + GC_FATAL("popped 0-word object"); + } + */ + + if (GC_BLOCK_PAGE(p)->kind != GC_PAGE_HUGE && !GET_BIT(block->debug_objects, obj_id)) { +//gc_debug_block(block); +// printf("popped: %p\n", p); fflush(stdout); +//gc_debug_obj(p); +//GC_FATAL("not an existing object!"); +// puts("non-ex"); +continue; + } + + if (block->huge_sized) { + words = (GC_BLOCK_PAGE(p)->size - 8192) / 8; + GC_DEBUG(mark, "scanning huge, %d words", words); + } else if (meta->medium_sized) { + words |= GC_METADATA_EXT(p) << 4; + words++; int last_id = GC_LINE_ID((char *)p + words * 8); int line_span = (last_id - line_id); GC_DEBUG(mark, "marking medium, %d lines", line_span); - memset(&block->line_marks[line_id], 1, line_span); - } else { - // for huge objects this does nothing useful +//printf("memset(%p, %d, %d) ... ", &block->line_marks[line_id], 1, line_span); fflush(stdout); + // memset(&block->line_marks[line_id], 1, line_span); + for (int i = line_id; i < last_id; i++) { + block->line_marks[i] = 1; + } +//puts("ok"); + } else if (!block->huge_sized) { + words++; block->line_marks[line_id] = 1; } - if (meta->no_ptr) { - continue; - } + gc_stats->live_memory_normal += words * 8; // scan object - void **data = (void **)((void *)p + sizeof(hl_type *)); - if (block->huge_sized) { - words = (GC_BLOCK_PAGE(p)->size - 8192) / 8; - GC_DEBUG(mark, "scanning huge, %d words", words); - } - // printf("%p %p %p %p -- type %s, %d words\n", p, meta, p->t, p->t->mark_bits, hl_type_name(p->t), words); - // printf("mark_bits 1 %d\n", p->t->mark_bits[0]); - // TODO: p->t->mark_bits should only be null with dynamic_mark - if (true) { //meta->dynamic_mark || p->t == NULL || p->t->mark_bits == NULL) { - for (int i = 0; i < words - 1; i++) { // skip hl_type *t - gc_push(data[i], &data[i]); + void **data = (void **)p; + if (meta->no_ptr) { + continue; + } else if (meta->raw || p->t == NULL || p->t->mark_bits == NULL || p->t->kind == HFUN) { + for (int i = 0; i < words; i++) { + gc_push(data[i], p); // &data[i]); } } else { - unsigned int *mark_bits = p->t->mark_bits; - for (int i = 0; i < words - 1; i++) { // skip hl_type *t - if (GET_BIT(mark_bits, i)) { - gc_push(data[i], &data[i]); + int pos = 0; + int *mark_bits = p->t->mark_bits; + if (p->t->kind == HENUM) { + venum *e = (venum *)p; + mark_bits = p->t->mark_bits + e->index; + data += 2; + words -= 2; + } else { + data++; + pos++; + } + for (; pos < words; pos++) { + if (GET_BIT32(mark_bits, pos)) { + gc_push(*data, p); // &data[i]); } + data++; } } } + gc_stats->live_memory += gc_stats->live_memory_normal; + GC_DEBUG_DUMP0("gc_mark.cleanup"); + dump_live(); // clean up stacks if (gc_mark_stack_active == &gc_mark_stack_next) { @@ -810,31 +1022,77 @@ GC_STATIC void gc_sweep(void) { // in each page, iterate occupied (block_bmp) pages // for each block // - count the number of used lines - // - finalise unmarked lines ? + // - finalise unmarked lines // - if compactable and fragmented, try compacting // - if empty, move into global free pool // - if not full but in full list, move into thread's recycle list - // normal page with blocks + // normal pages with blocks for (gc_page_header_t *page = gc_last_allocated_page; page != NULL; page = page->next_page) { for (int block_id = 0; block_id < GC_BLOCKS_PER_PAGE; block_id++) { if (GET_BIT(page->block_bmp, block_id)) { gc_block_header_t *block = GC_PAGE_BLOCK(page, block_id); - int lines_used = 0; + //if (block == GC_LINE_BLOCK(ID_TRACK(12, 0x188900))) { + // puts("DEBUG BLOCK"); + // gc_debug_obj(ID_TRACK(12, 0x188900)); + //} + int lines_used = (block->line_marks[0] ? 1 : 0); for (int line_id = 0; line_id < GC_LINES_PER_BLOCK; line_id++) { - if (block->line_marks[line_id]) { + gc_object_t *p = (gc_object_t *)&block->lines[line_id]; + gc_metadata_t *meta = &block->metadata[line_id * 16]; + if (!block->line_marks[line_id] && block->line_finalize[line_id]) { + void **f = p; + void (*finalize)(void *) = *((void (**)(void *))p); +/*if (page == tracked_page && ((int_val)p & 0x3FFFFF) == 0x2d2700) { + printf("before finalized!\n"); + gc_debug_obj(p); +}*/ +//if (p == ID_TRACK(9, 0x2b5c80)) { +// puts("DEBUG ABOUT TO FINALISE"); +// gc_debug_obj(p); +// printf("finalising block: %p, line: %d, obj: %p (page: %p, %ld), f: %p ... "/*" (*: %p) ... "*/, block, line_id, p, page, (int_val)p & 0x3FFFFF, finalize); //, finalize != NULL ? *(void **)(*f) : NULL); +// fflush(stdout); +//// gc_debug_interactive(); +//} +/*if (*(void **)(*f) == (void *)0x0B) { + gc_debug_obj(&block->lines[line_id]); + GC_FATAL("here!"); +} +fflush(stdout);*/ + if (finalize != NULL) { + finalize((void *)&block->lines[line_id]); + *f = NULL; + } + block->line_finalize[line_id] = 0; +//if (p == ID_TRACK(9, 0x2b5c80)) { +// puts("ok"); +// fflush(stdout); +//} + } + if (!block->line_marks[line_id] && block->debug_objects) { + for (int i = 0; i < 16; i++) { + int obj_id = line_id * 16 + i; + SET_BIT0(block->debug_objects, obj_id); + } + } + if (line_id == 0) + continue; + if (block->line_marks[line_id - 1] || block->line_marks[line_id]) { lines_used++; } } GC_DEBUG_DUMP2("gc_sweep.block", block, block->line_marks); GC_DEBUG(sweep, "block %p uses %d lines", block, lines_used); - if (lines_used == 0) { - gc_push_block(block); - } else if (lines_used < GC_LINES_PER_BLOCK && block->kind == GC_BLOCK_FULL) { - DLL_UNLINK(block); - block->kind = GC_BLOCK_RECYCLED; - // TODO: get thread by block->owner_thread - DLL_INSERT(block, current_thread->recyclable_blocks); + if (block->kind == GC_BLOCK_FULL) { + if (lines_used == 0) { + DLL_UNLINK(block); + gc_push_block(block); + } else if (lines_used < GC_LINES_PER_BLOCK) { + DLL_UNLINK(block); + block->kind = GC_BLOCK_RECYCLED; + // TODO: get thread by block->owner_thread + DLL_INSERT(block, current_thread->recyclable_blocks); + } } } } @@ -856,6 +1114,14 @@ GC_STATIC void gc_sweep(void) { // change polarity for the next cycle gc_mark_polarity = 1 - gc_mark_polarity; + // grow heap if needed +//printf("live memory normal: %lu\n", gc_stats->live_memory_normal); +//printf("total pages normal: %lu\n", gc_stats->total_pages_normal); + if (gc_stats->total_pages_normal < 90 && (float)(gc_stats->live_memory_normal / GC_PAGE_SIZE) > (float)gc_stats->total_pages_normal * gc_config->min_grow_usage) { + int growth = (int)((float)gc_stats->total_pages_normal * gc_config->heap_growth); + gc_grow_heap(growth); + } + gc_stats->cycles++; } @@ -878,7 +1144,8 @@ GC_STATIC gc_block_header_t *gc_get_block(void *p) { } HL_API void hl_blocking(bool b) { - // hl_fatal("hl_blocking not implemented"); + // TODO: threads + gc_blocked = b; } HL_API hl_thread_info *hl_get_thread() { @@ -911,11 +1178,9 @@ HL_API vdynamic *hl_alloc_obj( hl_type *t ) { size = rt->size; if( size & (HL_WSIZE-1) ) size += HL_WSIZE - (size & (HL_WSIZE-1)); if( t->kind == HSTRUCT ) { - o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? GC_ALLOC_RAW : GC_ALLOC_NOPTR); - o->t = NULL; // TODO: not needed??? (just pass NULL above?) + o = (vobj*)hl_gc_alloc_gen(NULL, size, rt->hasPtr ? GC_ALLOC_RAW : GC_ALLOC_NOPTR); } else { o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); - o->t = t; // TODO: not needed } for(i=0;inbindings;i++) { hl_runtime_binding *b = rt->bindings + i; @@ -952,9 +1217,19 @@ HL_API void hl_gc_set_dump_types( hl_types_dump tdump ) { // GC initialisation ----------------------------------------------- +#include +GC_STATIC void gc_handle_signal(int signum) { + signal(signum, SIG_DFL); + printf("SIGNAL %d\n", signum); + // hl_dump_stack(); + gc_debug_interactive(); + raise(signum); +} + GC_STATIC gc_stats_t *gc_init(void) { gc_mutex_roots = hl_mutex_alloc(false); gc_mutex_pool = hl_mutex_alloc(false); + gc_blocked = false; base_addr = (void *)0x40000000; gc_min_allocated = (void *)0xFFFFFFFFFFFFFFFF; @@ -982,10 +1257,15 @@ GC_STATIC gc_stats_t *gc_init(void) { gc_stats->live_blocks = 0; gc_stats->total_memory = 0; gc_stats->cycles = 0; + gc_stats->live_memory = 0; + gc_stats->live_memory_normal = 0; gc_stats->total_pages = 0; + gc_stats->total_pages_normal = 0; gc_config = (gc_config_t *)calloc(sizeof(gc_config_t), 1); - // gc_config->memory_limit = 500 * 1024 * 1024; - gc_config->memory_limit = 100 * 1024 * 1024; + gc_config->min_heap_pages = 10; + gc_config->memory_limit = 500 * 1024 * 1024; + gc_config->heap_growth = 0.35f; + gc_config->min_grow_usage = 0.2f; gc_config->debug_fatal = true; gc_config->debug_os = false; gc_config->debug_page = false; @@ -993,8 +1273,19 @@ GC_STATIC gc_stats_t *gc_init(void) { gc_config->debug_mark = false; gc_config->debug_sweep = false; gc_config->debug_other = false; - gc_config->debug_dump = true; + gc_config->debug_dump = false; gc_config->dump_file = fopen("gc.dump", "w"); + //dump_file = fopen("gc.live", "w"); + gc_grow_heap(gc_config->min_heap_pages); + + //struct sigaction act; + //act.sa_sigaction = NULL; + //act.sa_handler = gc_handle_signal; + //act.sa_flags = 0; + //sigemptyset(&act.sa_mask); + //sigaction(SIGILL,&act,NULL); + //sigaction(SIGSEGV,&act,NULL); + return gc_stats; } @@ -1029,6 +1320,309 @@ void hl_global_init() { } void hl_global_free() { + printf("%lu\n", gc_stats->cycles); gc_deinit(); hl_cache_free(); } + +void gc_debug_block(gc_block_header_t *block) { + static char *kind_names[] = { + "(not initialised)", + "free", + "new", + "full", + "recycled", + "zombie", + "brand new" + }; + if (block == NULL) { + puts("block: NULL"); + return; + } + // printf("block: %p, kind: %s, prev: %p, next: %p\n", block, kind_names[block->kind], block->prev, block->next); + printf("block: %p\n", block); + gc_page_header_t *page = (gc_page_header_t *)block; + printf(" (P) next_page: %p\n", page->next_page); + printf(" (P) next_page_bucket: %p\n", page->next_page_bucket); + printf(" (P) block_bmp: %lx\n", *((long int *)(&page->block_bmp))); + printf(" (P) size: %d\n", page->size); + printf(" (P) free_blocks: %d\n", (int)page->free_blocks); + printf(" (P) kind: %d\n", (int)page->kind); + printf(" (B) prev: %p\n", block->prev); + printf(" (B) next: %p\n", block->next); + printf(" (B) kind: %s\n", kind_names[block->kind]); + printf(" (B) owner_thread: %d\n", block->owner_thread); + printf(" (B) huge_sized: %d\n", block->huge_sized); + + bool last_used = false; + for (int line = 0; line < GC_LINES_PER_BLOCK; line++) { + if (line % 128 == 0) + printf(" "); + if (block->line_marks[line]) + printf("X"); + else + printf("."); + if (line % 128 == 127) + puts(""); + } + /* + puts(""); + if (block->debug_objects != NULL) { + for (int obj = 0; obj < 448 * 16; obj++) { + if (obj % 200 == 0) + printf(" "); + if (GET_BIT(block->debug_objects, obj)) + printf("X"); + else + printf("."); + if (obj % 200 == 199) + puts(""); + } + }*/ + puts(""); + int lines_used = (block->line_marks[0] ? 1 : 0); + for (int line_id = 1; line_id < GC_LINES_PER_BLOCK; line_id++) { + if (block->line_marks[line_id - 1] || block->line_marks[line_id]) { + lines_used++; + } + } + printf("lines used: %d\n", lines_used); +} + +void gc_debug_obj_id(void **p) { + gc_block_header_t *block = GC_LINE_BLOCK(p); + gc_page_header_t *page = GC_BLOCK_PAGE(block); + for (int i = 0; i < page_counter && i < 500; i++) { + if (page == tracked_page_addr[i]) { + printf("%p ID_TRACK(%d, 0x%x)\n", p, i, (int_val)p & 0x3FFFFF); fflush(stdout); + return; + } + } + for (int i = 0; i < huge_counter && i < 500; i++) { + if (page == tracked_huge_addr[i]) { + printf("%p HUGE ID_TRACK(%d, 0x%x)\n", p, i, (int_val)p & 0x3FFFFF); fflush(stdout); + return; + } + } + printf("%p ID_???\n", p); fflush(stdout); +} + +void gc_debug_obj(void **p) { + gc_debug_obj_id(p); + if (p != NULL) { + gc_block_header_t *block = GC_LINE_BLOCK(p); + gc_metadata_t *meta = GC_METADATA(p); + puts("meta:"); + int real_words = meta->words; + printf(" words: %d (+ 1)\n", meta->words); + printf(" marked: %d\n", meta->marked); + printf(" medium_sized: %d\n", meta->medium_sized); + printf(" raw: %d\n", meta->raw); + printf(" no_ptr: %d\n", meta->no_ptr); + if (meta->medium_sized) { + printf(" (ext) words: %d\n", GC_METADATA_EXT(p)); + real_words |= GC_METADATA_EXT(p) << 4; + } + if ((int_val)p & 0xFFFFFFFFFFFFFF80 == (int_val)p) { + printf("finalize: %d\n", block->line_finalize[GC_LINE_ID(p)]); + } + real_words++; + hl_type *t = *p; + printf("type: %p\n", t); + /* + void **data = (void **)p; + for (int i = 0; i < real_words; i++) { + printf(" - %d (at %p): %p\n", i, &data[i], data[i]); + } + */ + /*if (t != NULL && t->mark_bits != NULL) { + printf(" mark bits: %p\n", t->mark_bits); + //if (t->mark_bits < 0x200000000) { + for (int i = 0; i < real_words; i++) { + if (GET_BIT32(t->mark_bits, i)) { + printf(" - %d\n", i); //": %p\n", i, *(void **)(p[i])); + } + } + //} + }*/ + printf("block: %p\n", block); + gc_debug_block(block); + fflush(stdout); + } +} + +static const uchar *TSTR[] = { + USTR("void"), USTR("i8"), USTR("i16"), USTR("i32"), USTR("i64"), USTR("f32"), USTR("f64"), + USTR("bool"), USTR("bytes"), USTR("dynamic"), NULL, NULL, + USTR("array"), USTR("type"), NULL, NULL, USTR("dynobj"), + NULL, NULL, NULL, NULL, NULL +}; +typedef struct tlist { + hl_type *t; + struct tlist *next; +} tlist; +typedef struct _stringitem { + uchar *str; + int size; + int len; + struct _stringitem *next; +} * stringitem; +struct hl_buffer { + int totlen; + int blen; + stringitem data; +}; +void hl_type_str_rec( hl_buffer *b, hl_type *t, tlist *parents ); + +void gc_debug_type(hl_type *t) { + uchar buf_data[1024] = {0}; + struct _stringitem buf_item = { + .str = buf_data, + .size = 1024, + .len = 0 + }; + struct hl_buffer buf = { + .totlen = 0, + .blen = 1024, + .data = &buf_item + }; + + const uchar *c = TSTR[t->kind]; + hl_buffer *b; + if (c != NULL) { + uprintf(USTR("%s\n"), c); + } else { + hl_type_str_rec(&buf, t, NULL); + uprintf(USTR("%s\n"), buf_data); + buf_data[0] = 0; + buf_item.len = 0; + buf.totlen = 0; + } +} + +void gc_debug_verify_pool(const char *at) { + gc_block_header_t *last, *cur; + + if (current_thread && current_thread->full_blocks) { + last = NULL; + cur = current_thread->full_blocks; + GC_ASSERT(cur == ¤t_thread->sentinels[0]); + while (cur != NULL) { + GC_ASSERT(cur->prev == last); + if (last != NULL && cur->next != NULL) { + GC_ASSERT_M(cur->kind == GC_BLOCK_FULL, "%p (%s)", cur, at); + } + last = cur; + cur = cur->next; + } + GC_ASSERT(last == ¤t_thread->sentinels[1]); + } + + if (current_thread && current_thread->recyclable_blocks) { + last = NULL; + cur = current_thread->recyclable_blocks; + GC_ASSERT(cur == ¤t_thread->sentinels[2]); + while (cur != NULL) { + GC_ASSERT(cur->prev == last); + if (last != NULL && cur->next != NULL) { + GC_ASSERT_M(cur->kind == GC_BLOCK_RECYCLED, "%p", cur); + } + last = cur; + cur = cur->next; + } + GC_ASSERT(last == ¤t_thread->sentinels[3]); + } + + last = NULL; + cur = gc_pool; + for (int i = 0; i < gc_pool_count; i++) { + GC_ASSERT_M(cur != NULL, "found NULL (%d)", i); + GC_ASSERT_M(cur >= gc_min_allocated && cur <= gc_max_allocated, "found unallocated (%p -> %p at %d)", last, cur, i); + last = cur; + cur = cur->next; + } + GC_ASSERT(cur == NULL); +} + +void *gc_debug_track_id(int pagenum, int low) { + return ID_TRACK(pagenum, low); +} +/* +void gc_debug_enable(bool dl) { + do_log = dl; +} +*/ + +void gc_debug_thread(void) { + hl_thread_info *t = current_thread; + printf("info: %p\n", t); + if (t != NULL) { + printf(" thread_id: %d\n", t->thread_id); + printf(" gc_blocking: %d\n", t->gc_blocking); + printf(" stack_top: %p\n", t->stack_top); + printf(" stack_cur: %p\n", t->stack_cur); + printf(" lines_start: %p\n", t->lines_start); + printf(" lines_limit: %p\n", t->lines_limit); + printf(" sentinels: %p\n", t->sentinels); + printf(" lines_block: %p\n", t->lines_block); + printf(" full_blocks: %p\n", t->full_blocks); + printf(" recyclable_blocks: %p\n", t->recyclable_blocks); + } +} + +void gc_debug_interactive(void) { + puts("gc interactive debugger (h for help)"); + bool run = true; + while (run) { + printf("> "); fflush(stdout); + char cmd; + scanf(" %c", &cmd); + switch (cmd) { + case 'h': + puts("h help"); + puts("b debug block"); + puts("o debug object"); + puts("O debug object (by ID_TRACK)"); + puts("s dump call stack"); + puts("t dump thread info"); + puts("T print type name"); + puts("q quit"); + break; + case 'b': { + gc_block_header_t *block; + if (scanf("%lx", &block) && block != NULL) { + gc_debug_block(block); + } + } break; + case 'o': { + gc_object_t *obj; + if (scanf("%lx", &obj) && obj != NULL) { + gc_debug_obj(obj); + } + } break; + case 'O': { + int pagenum; + int low; + if (scanf("%d %d", &pagenum, &low) && pagenum != 0 && low != 0) { + gc_debug_obj(ID_TRACK(pagenum, low)); + } + } break; + case 's': + hl_dump_stack(); + break; + case 't': + gc_debug_thread(); + break; + case 'T': { + hl_type *t; + if (scanf("%lx", &t) && t != NULL) { + gc_debug_type(t); + } + } break; + case 'q': { + run = false; + } break; + default: break; + } + } +} diff --git a/src/gc.h b/src/gc.h index cfe2008db..3cc4791a2 100644 --- a/src/gc.h +++ b/src/gc.h @@ -69,6 +69,8 @@ typedef enum { GC_BLOCK_RECYCLED, // TODO: zombie blocks GC_BLOCK_ZOMBIE, + // just popped from OS page: no pointers + GC_BLOCK_BRAND_NEW, _force_int = 0x7FFFFFFF } gc_block_type_t; @@ -82,17 +84,23 @@ typedef union { // uint8_t huge_sized : 1; // when set, object cannot be moved // uint8_t pinned : 1; + /* // whether all words of the object are potentially pointers // (set for dynamics and pointer arrays) uint8_t dynamic_mark : 1; + */ + // TODO: + uint8_t raw : 1; // when set, no pointers are traced in the object uint8_t no_ptr : 1; - // number of words (sizeof(void *)) + // number of words (sizeof(void *)) - 1 + // (i.e. words == 0 means 8 bytes, words == 0xFF means 128 bytes) uint8_t words : 4; }; uint8_t flags; } gc_metadata_t; +/* typedef union { struct { gc_metadata_t base; @@ -100,6 +108,7 @@ typedef union { }; uint16_t flags; } gc_metadata_ext_t; +*/ typedef struct gc_block_header_s { gc_page_header_t page_header; // only set in the first block of page @@ -111,7 +120,9 @@ typedef struct gc_block_header_s { unsigned char _pad[4]; unsigned char line_marks[GC_LINES_PER_BLOCK]; gc_metadata_t metadata[GC_LINES_PER_BLOCK * 16]; - unsigned char _pad2[512]; + char *debug_objects; + unsigned char line_finalize[GC_LINES_PER_BLOCK]; + unsigned char _pad2[56]; gc_line_t lines[GC_LINES_PER_BLOCK]; } gc_block_header_t; @@ -164,11 +175,17 @@ typedef struct { unsigned long live_blocks; unsigned long total_memory; unsigned long cycles; + unsigned long live_memory; + unsigned long live_memory_normal; int total_pages; + int total_pages_normal; } gc_stats_t; typedef struct { + unsigned long min_heap_pages; unsigned long memory_limit; + float heap_growth; + float min_grow_usage; bool debug_fatal; bool debug_os; bool debug_page; @@ -187,6 +204,8 @@ GC_STATIC gc_page_header_t *gc_alloc_page_memory(void); // frees a GC page GC_STATIC void gc_free_page_memory(gc_page_header_t *header); +GC_STATIC void gc_grow_heap(int count); + // gets the `block`th block in `page` #define GC_PAGE_BLOCK(page, block) ((gc_block_header_t *)((char *)(page) + (block) * GC_BLOCK_SIZE)) // gets the page to which `block` belongs @@ -199,6 +218,7 @@ GC_STATIC void gc_free_page_memory(gc_page_header_t *header); #define GC_LINE_ID(ptr) ((int)((int_val)(ptr) - (int_val)GC_LINE_BLOCK(ptr)) / GC_LINE_SIZE - 64) #define GC_METADATA(obj) (&(GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024])) +#define GC_METADATA_EXT(obj) *(unsigned char *)(&GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024 + 1]) // roots ----------------------------------------------------------- @@ -234,9 +254,9 @@ GC_STATIC gc_block_header_t *gc_get_block(void *); // NULL if not a GC pointer // GC debug #if GC_ENABLE_DEBUG -# define GC_DEBUG(stream, f, ...) do { if (gc_config.debug_ ## stream) printf("[" #stream "] " f "\n", ##__VA_ARGS__); } while (0) -# define GC_DEBUG_DUMP_S(id) do { if (gc_config.debug_dump) { fwrite((id), 1, strlen(id) + 1, gc_config.dump_file); fflush(gc_config.dump_file); } } while (0) -# define GC_DEBUG_DUMP_R(arg) do { if (gc_config.debug_dump) { fwrite(&(arg), 1, sizeof(arg), gc_config.dump_file); fflush(gc_config.dump_file); } } while (0) +# define GC_DEBUG(stream, f, ...) do { if (gc_config->debug_ ## stream) printf("[" #stream "] " f "\n", ##__VA_ARGS__); } while (0) +# define GC_DEBUG_DUMP_S(id) do { if (gc_config->debug_dump) { fwrite((id), 1, strlen(id) + 1, gc_config->dump_file); fflush(gc_config->dump_file); } } while (0) +# define GC_DEBUG_DUMP_R(arg) do { if (gc_config->debug_dump) { fwrite(&(arg), 1, sizeof(arg), gc_config->dump_file); fflush(gc_config->dump_file); } } while (0) # define GC_DEBUG_DUMP0(id) do { GC_DEBUG_DUMP_S(id); int s = 0; GC_DEBUG_DUMP_R(s); } while (0) # define GC_DEBUG_DUMP1(id, arg1) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); } while (0) # define GC_DEBUG_DUMP2(id, arg1, arg2) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1) + sizeof(arg2); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); GC_DEBUG_DUMP_R(arg2); } while (0) @@ -250,10 +270,30 @@ GC_STATIC gc_block_header_t *gc_get_block(void *); // NULL if not a GC pointer # define GC_DEBUG_DUMP2(...) # define GC_DEBUG_DUMP3(...) #endif -#define GC_FATAL(msg) do { puts(msg); __builtin_trap(); } while (0) +#define GC_FATAL(msg) do { puts(msg); gc_debug_interactive(); __builtin_trap(); } while (0) +#define GC_ASSERT(cond) do { \ + if (!(cond)) { \ + printf("\x1B[38;5;160mGC assertion failed (line %d): " #cond "\n\x1B[0m", __LINE__); \ + GC_FATAL("!"); \ + } \ + } while (0) +#define GC_ASSERT_M(cond, msg, ...) do { \ + if (!(cond)) { \ + printf("\x1B[38;5;160mGC assertion failed (line %d): " #cond "\n\x1B[0m", __LINE__); \ + printf("info: " msg "\n", ##__VA_ARGS__); \ + GC_FATAL("!"); \ + } \ + } while (0) -/* GC_STATIC void gc_debug_block(gc_block_header_t *block); +GC_STATIC void gc_debug_obj(void **p); +GC_STATIC void gc_debug_obj_id(void **p); +GC_STATIC void gc_debug_type(hl_type *t); +GC_STATIC void gc_debug_interactive(void); +GC_STATIC void gc_debug_verify_pool(const char *); +GC_STATIC void *gc_debug_track_id(int, int); +//GC_STATIC void gc_debug_enable(bool); +/* GC_STATIC void gc_debug_general(void); GC_STATIC void gc_dump(const char *); */ diff --git a/src/hl.h b/src/hl.h index d83010dad..6f616f53d 100644 --- a/src/hl.h +++ b/src/hl.h @@ -688,9 +688,13 @@ HL_API void hl_tls_free( hl_tls *l ); // ----------------------- GC ALLOC ----------------------------------------------- -#define GC_ALLOC_DYNAMIC 1 -#define GC_ALLOC_NOPTR 2 -#define GC_ALLOC_RAW 4 +#define GC_ALLOC_FLAG_RAW 1 +#define GC_ALLOC_FLAG_NOPTR 2 + +#define GC_ALLOC_DYNAMIC 0 +#define GC_ALLOC_NOPTR (GC_ALLOC_FLAG_NOPTR) +#define GC_ALLOC_RAW (GC_ALLOC_FLAG_RAW) +#define GC_ALLOC_FINALIZER (GC_ALLOC_FLAG_NOPTR | GC_ALLOC_FLAG_RAW) HL_API void *hl_gc_alloc_gen( hl_type *t, int size, int flags ); HL_API void hl_add_root( void **ptr ); @@ -706,10 +710,9 @@ HL_API void hl_gc_set_dump_types( hl_types_dump tdump ); #define hl_gc_alloc_noptr(size) hl_gc_alloc_gen(&hlt_bytes, size, GC_ALLOC_NOPTR) #define hl_gc_alloc(t, size) hl_gc_alloc_gen(t, size, GC_ALLOC_DYNAMIC) -#define hl_gc_alloc_raw(size) hl_gc_alloc_gen(&hlt_abstract, size, 0) -// TODO: MEM_KIND_RAW) -#define hl_gc_alloc_finalizer(size) hl_gc_alloc_gen(&hlt_abstract, size, 0) -// TODO: MEM_KIND_FINALIZER) +#define hl_gc_alloc_raw(size) \ + (/*printf("allow_raw: %s %d\n", __FILE__, __LINE__), */hl_gc_alloc_gen(&hlt_abstract, size, GC_ALLOC_RAW)) +#define hl_gc_alloc_finalizer(size) hl_gc_alloc_gen(&hlt_abstract, size, GC_ALLOC_FINALIZER) // ----------------------- INTERNAL ALLOC ----------------------------------------- @@ -856,6 +859,8 @@ typedef struct gc_block_dummy_s gc_block_dummy_t; typedef struct gc_block_header_s gc_block_header_t; typedef struct { + jmp_buf gc_regs; + int thread_id; // gc vars @@ -876,8 +881,8 @@ typedef struct { vdynamic *exc_value; int flags; int exc_stack_count; + // extra - jmp_buf gc_regs; void *exc_stack_trace[HL_EXC_MAX_STACK]; } hl_thread_info; diff --git a/src/std/types.c b/src/std/types.c index 50f964251..7ad1a8f58 100644 --- a/src/std/types.c +++ b/src/std/types.c @@ -237,7 +237,7 @@ typedef struct tlist { struct tlist *next; } tlist; -static void hl_type_str_rec( hl_buffer *b, hl_type *t, tlist *parents ) { +/*static*/ void hl_type_str_rec( hl_buffer *b, hl_type *t, tlist *parents ) { const uchar *c = TSTR[t->kind]; tlist *l, cur; int i; From ba41c3e4604f1b8874611546512379c877f4638e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Tue, 28 Apr 2020 12:21:15 +0100 Subject: [PATCH 04/13] rename flags back to MEM_* --- src/gc.c | 22 +++++++++++----------- src/hl.h | 23 +++++++++++------------ src/std/array.c | 2 +- src/std/types.c | 2 +- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/gc.c b/src/gc.c index 6703d71b6..9a139c382 100644 --- a/src/gc.c +++ b/src/gc.c @@ -557,10 +557,10 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) meta->marked = !gc_mark_polarity; int sub_words = words - 1; meta->words = sub_words & 15; - if (flags & GC_ALLOC_FLAG_RAW) { + if (flags & MEM_KIND_RAW) { meta->raw = 1; } - if (flags & GC_ALLOC_FLAG_NOPTR) { + if (flags & MEM_KIND_NOPTR) { meta->no_ptr = 1; } // int line_id = GC_LINE_ID(ret); @@ -577,7 +577,7 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) // GC_DEBUG("allocated medium %p in block %p (%d bytes, %d lines, %d words)", ret, block, size, ret->line_span, words); } - if (flags == GC_ALLOC_FINALIZER) { + if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { // TODO: also in a finaliser-specific alloc function (after alloc_gen call) block->line_finalize[GC_LINE_ID(ret)] = 1; } @@ -660,7 +660,7 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { // align to words int words = (size + (sizeof(void *) - 1)) / sizeof(void *); - if (flags == GC_ALLOC_FINALIZER) { + if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { // TODO: separate function to do this before alloc_gen // (to reduce hot path for non-finaliser objects) @@ -721,7 +721,7 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { return gc_alloc_bump(t, size, words, flags); } else { // TODO: separate path for huge objects -if (flags == GC_ALLOC_FINALIZER) { +if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { GC_FATAL("cannot alloc huge finalizer"); } gc_object_t *obj = gc_pop_huge(size); @@ -729,10 +729,10 @@ if (flags == GC_ALLOC_FINALIZER) { gc_metadata_t *meta = GC_METADATA(obj); meta->flags = 0; meta->marked = !gc_mark_polarity; - if (flags & GC_ALLOC_FLAG_RAW) { + if (flags & MEM_KIND_RAW) { meta->raw = 1; } - if (flags & GC_ALLOC_FLAG_NOPTR) { + if (flags & MEM_KIND_NOPTR) { meta->no_ptr = 1; } gc_stats->live_memory += size; @@ -1159,7 +1159,7 @@ HL_API bool hl_is_gc_ptr(void *ptr) { // HL_API varray *hl_alloc_array( hl_type *t, int size ) { return NULL; } HL_API vdynamic *hl_alloc_dynamic( hl_type *t ) { - return (vdynamic*)hl_gc_alloc_gen(t, sizeof(vdynamic), hl_is_ptr(t) ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); + return (vdynamic*)hl_gc_alloc_gen(t, sizeof(vdynamic), hl_is_ptr(t) ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR); } static const vdynamic vdyn_true = { &hlt_bool, {true} }; @@ -1178,9 +1178,9 @@ HL_API vdynamic *hl_alloc_obj( hl_type *t ) { size = rt->size; if( size & (HL_WSIZE-1) ) size += HL_WSIZE - (size & (HL_WSIZE-1)); if( t->kind == HSTRUCT ) { - o = (vobj*)hl_gc_alloc_gen(NULL, size, rt->hasPtr ? GC_ALLOC_RAW : GC_ALLOC_NOPTR); + o = (vobj*)hl_gc_alloc_gen(NULL, size, rt->hasPtr ? MEM_KIND_RAW : MEM_KIND_NOPTR); } else { - o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); + o = (vobj*)hl_gc_alloc_gen(t, size, rt->hasPtr ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR); } for(i=0;inbindings;i++) { hl_runtime_binding *b = rt->bindings + i; @@ -1205,7 +1205,7 @@ HL_API vvirtual *hl_alloc_virtual( hl_type *t ) { } HL_API vdynobj *hl_alloc_dynobj(void) { - return (vdynobj*)hl_gc_alloc_gen(&hlt_dynobj,sizeof(vdynobj), GC_ALLOC_DYNAMIC); + return (vdynobj*)hl_gc_alloc_gen(&hlt_dynobj,sizeof(vdynobj), MEM_KIND_DYNAMIC); } // HL_API vbyte *hl_alloc_bytes( int size ) { return NULL; } diff --git a/src/hl.h b/src/hl.h index 6f616f53d..4a01fc2c7 100644 --- a/src/hl.h +++ b/src/hl.h @@ -688,13 +688,13 @@ HL_API void hl_tls_free( hl_tls *l ); // ----------------------- GC ALLOC ----------------------------------------------- -#define GC_ALLOC_FLAG_RAW 1 -#define GC_ALLOC_FLAG_NOPTR 2 - -#define GC_ALLOC_DYNAMIC 0 -#define GC_ALLOC_NOPTR (GC_ALLOC_FLAG_NOPTR) -#define GC_ALLOC_RAW (GC_ALLOC_FLAG_RAW) -#define GC_ALLOC_FINALIZER (GC_ALLOC_FLAG_NOPTR | GC_ALLOC_FLAG_RAW) +#define MEM_HAS_PTR (!((kind)&2)) +#define MEM_KIND_DYNAMIC 0 +#define MEM_KIND_RAW 1 +#define MEM_KIND_NOPTR 2 +#define MEM_KIND_FINALIZER 3 +#define MEM_ALIGN_DOUBLE 128 +#define MEM_ZERO 256 HL_API void *hl_gc_alloc_gen( hl_type *t, int size, int flags ); HL_API void hl_add_root( void **ptr ); @@ -708,11 +708,10 @@ HL_API bool hl_is_blocking( void ); typedef void (*hl_types_dump)( void (*)( void *, int) ); HL_API void hl_gc_set_dump_types( hl_types_dump tdump ); -#define hl_gc_alloc_noptr(size) hl_gc_alloc_gen(&hlt_bytes, size, GC_ALLOC_NOPTR) -#define hl_gc_alloc(t, size) hl_gc_alloc_gen(t, size, GC_ALLOC_DYNAMIC) -#define hl_gc_alloc_raw(size) \ - (/*printf("allow_raw: %s %d\n", __FILE__, __LINE__), */hl_gc_alloc_gen(&hlt_abstract, size, GC_ALLOC_RAW)) -#define hl_gc_alloc_finalizer(size) hl_gc_alloc_gen(&hlt_abstract, size, GC_ALLOC_FINALIZER) +#define hl_gc_alloc_noptr(size) hl_gc_alloc_gen(&hlt_bytes, size, MEM_KIND_NOPTR) +#define hl_gc_alloc(t, size) hl_gc_alloc_gen(t, size, MEM_KIND_DYNAMIC) +#define hl_gc_alloc_raw(size) hl_gc_alloc_gen(&hlt_abstract, size, MEM_KIND_RAW) +#define hl_gc_alloc_finalizer(size) hl_gc_alloc_gen(&hlt_abstract, size, MEM_KIND_FINALIZER) // ----------------------- INTERNAL ALLOC ----------------------------------------- diff --git a/src/std/array.c b/src/std/array.c index 7b7f7f327..1c6db62b4 100644 --- a/src/std/array.c +++ b/src/std/array.c @@ -25,7 +25,7 @@ HL_PRIM varray *hl_alloc_array( hl_type *at, int size ) { int esize = hl_type_size(at); varray *a; if( size < 0 ) hl_error("Invalid array size"); - a = (varray*)hl_gc_alloc_gen(&hlt_array, sizeof(varray) + esize*size, hl_is_ptr(at) ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); + a = (varray*)hl_gc_alloc_gen(&hlt_array, sizeof(varray) + esize*size, (hl_is_ptr(at) ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR) | MEM_ZERO); a->at = at; a->size = size; return a; diff --git a/src/std/types.c b/src/std/types.c index 7ad1a8f58..4439318dd 100644 --- a/src/std/types.c +++ b/src/std/types.c @@ -530,7 +530,7 @@ HL_PRIM bool hl_type_enum_eq( venum *a, venum *b ) { HL_PRIM venum *hl_alloc_enum( hl_type *t, int index ) { hl_enum_construct *c = t->tenum->constructs + index; - venum *v = (venum*)hl_gc_alloc_gen(t, c->size, c->hasptr ? GC_ALLOC_DYNAMIC : GC_ALLOC_NOPTR); + venum *v = (venum*)hl_gc_alloc_gen(t, c->size, MEM_KIND_DYNAMIC | (c->hasptr ? 0 : MEM_KIND_NOPTR) | MEM_ZERO); v->index = index; return v; } From d90626cec04221e3a559d5dfa2d072ef6c54e1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Tue, 28 Apr 2020 12:45:14 +0100 Subject: [PATCH 05/13] fix medium marking at the end of block --- src/gc.c | 15 +++++---------- src/gc.h | 4 +++- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/gc.c b/src/gc.c index 9a139c382..dc4671ef8 100644 --- a/src/gc.c +++ b/src/gc.c @@ -925,7 +925,7 @@ GC_STATIC void gc_mark(void) { // puts("popped lock"); // mark line(s) - int line_id = GC_LINE_ID(p); + int line_id = GC_LINE_ID_IN(p, block); int obj_id = ((int_val)p - (int_val)&block->lines[0]) / 8; int words = meta->words; @@ -944,8 +944,8 @@ GC_STATIC void gc_mark(void) { // printf("popped: %p\n", p); fflush(stdout); //gc_debug_obj(p); //GC_FATAL("not an existing object!"); -// puts("non-ex"); -continue; + printf("non-ex %p\n", p); + continue; } if (block->huge_sized) { @@ -954,15 +954,10 @@ continue; } else if (meta->medium_sized) { words |= GC_METADATA_EXT(p) << 4; words++; - int last_id = GC_LINE_ID((char *)p + words * 8); + int last_id = GC_LINE_ID_IN((char *)p + words * 8, block); int line_span = (last_id - line_id); GC_DEBUG(mark, "marking medium, %d lines", line_span); -//printf("memset(%p, %d, %d) ... ", &block->line_marks[line_id], 1, line_span); fflush(stdout); - // memset(&block->line_marks[line_id], 1, line_span); - for (int i = line_id; i < last_id; i++) { - block->line_marks[i] = 1; - } -//puts("ok"); + memset(&block->line_marks[line_id], 1, line_span); } else if (!block->huge_sized) { words++; block->line_marks[line_id] = 1; diff --git a/src/gc.h b/src/gc.h index 3cc4791a2..82465367e 100644 --- a/src/gc.h +++ b/src/gc.h @@ -214,8 +214,10 @@ GC_STATIC void gc_grow_heap(int count); #define GC_LINE_BLOCK(line) ((gc_block_header_t *)((int_val)(line) & (int_val)(~(GC_BLOCK_SIZE - 1)))) // gets the index of `block` in its page #define GC_BLOCK_ID(block) ((int)(((int_val)(block) - (int_val)GC_BLOCK_PAGE(block)) / GC_BLOCK_SIZE)) +// gets the line index of `ptr` in the given block +#define GC_LINE_ID_IN(ptr, block) ((int)((int_val)(ptr) - (int_val)(block)) / GC_LINE_SIZE - 64) // gets the line index of `ptr` in its block -#define GC_LINE_ID(ptr) ((int)((int_val)(ptr) - (int_val)GC_LINE_BLOCK(ptr)) / GC_LINE_SIZE - 64) +#define GC_LINE_ID(ptr) GC_LINE_ID_IN(ptr, GC_LINE_BLOCK(ptr)) #define GC_METADATA(obj) (&(GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024])) #define GC_METADATA_EXT(obj) *(unsigned char *)(&GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024 + 1]) From b34e5c32c264f840b62f4d336e211d5f747050d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Tue, 28 Apr 2020 15:19:36 +0100 Subject: [PATCH 06/13] remove stale debug comments, make debug macros no-ops without GC_ENABLE_DEBUG --- src/gc.c | 158 ++++++++++--------------------------------------------- src/gc.h | 36 ++++++++----- 2 files changed, 51 insertions(+), 143 deletions(-) diff --git a/src/gc.c b/src/gc.c index dc4671ef8..3685ed84f 100644 --- a/src/gc.c +++ b/src/gc.c @@ -57,16 +57,16 @@ GC_ASSERT_M((obj)->next != NULL, "%p", (obj)); \ GC_ASSERT_M((obj)->prev != NULL, "%p", (obj)); \ (obj)->next->prev = (obj)->prev; \ - /* printf("%p->n := %p\n", (obj)->prev, (obj)->next); */ (obj)->prev->next = (obj)->next; \ - /* printf("%p->n := NULL\n", (obj)->next); */ (obj)->next = NULL; \ + (obj)->prev->next = (obj)->next; \ + (obj)->next = NULL; \ (obj)->prev = NULL; \ } while (0) #define DLL_INSERT(obj, head) do { \ GC_ASSERT((obj) != NULL); \ GC_ASSERT((head) != NULL); \ - /* printf("%p->n := %p\n", (obj), (head)->next); */ (obj)->next = (head)->next; \ + (obj)->next = (head)->next; \ (obj)->prev = (head); \ - /* printf("%p->n := %p\n", (head), (obj)); */ (head)->next = (obj); \ + (head)->next = (obj); \ (obj)->next->prev = (obj); \ } while (0) @@ -84,7 +84,6 @@ static int page_counter = 0; static int huge_counter = 0; static gc_page_header_t *tracked_page_addr[500] = {0}; static gc_page_header_t *tracked_huge_addr[500] = {0}; -// static bool do_log = false; #define ID_TRACK(page_num, low) \ (gc_object_t *)((int_val)tracked_page_addr[page_num] | ((int_val)low & 0x3FFFFF)) @@ -162,8 +161,8 @@ GC_STATIC void gc_add_page(gc_page_header_t *header) { if ((char *)header < (char *)gc_min_allocated) { gc_min_allocated = (char *)header; } - if ((char *)header + GC_PAGE_SIZE > (char *)gc_max_allocated) { // TODO: this over-estimates max alloc - gc_max_allocated = (char *)header + GC_PAGE_SIZE; + if ((char *)header + header->size > (char *)gc_max_allocated) { + gc_max_allocated = (char *)header + header->size; } gc_stats->total_pages++; @@ -223,7 +222,6 @@ if (page_counter < 500) gc_pool = &blocks[0]; gc_pool_count += GC_BLOCKS_PER_PAGE; hl_mutex_release(gc_mutex_pool); -//printf("PAGE %p\n", header); gc_debug_verify_pool("page"); GC_DEBUG_DUMP1("gc_alloc_page_normal.success", header); @@ -232,7 +230,6 @@ gc_debug_verify_pool("page"); // allocates a huge page (containing a single object) GC_STATIC gc_page_header_t *gc_alloc_page_huge(int size) { -//printf("HUGE before\n"); gc_debug_verify_pool("huge before"); gc_page_header_t *header = (gc_page_header_t *)gc_alloc_os_memory(sizeof(gc_page_header_t) + size); @@ -250,7 +247,6 @@ gc_debug_verify_pool("huge before"); if (huge_counter < 500) tracked_huge_addr[huge_counter++] = header; -//printf("HUGE %p\n", header); gc_debug_verify_pool("huge after"); GC_DEBUG_DUMP2("gc_alloc_page_huge.success", header, size); @@ -456,9 +452,7 @@ gc_debug_verify_pool("pop before"); if (UNLIKELY(gc_pool_count == 0)) { // if (gc_stats->total_memory * 1.2 >= gc_config->memory_limit) { if (gc_stats->live_blocks > 0) { - GC_DEBUG(alloc, "using %lu / %lu bytes", gc_stats->total_memory, gc_config->memory_limit); - // GC_FATAL("OOM: memory limit hit"); - GC_DEBUG(alloc, "memory limit hit, triggering major ..."); + GC_DEBUG(alloc, "using %lu / %lu bytes, triggering major ...", gc_stats->total_memory, gc_config->memory_limit); hl_gc_major(); } if (gc_pool_count == 0 && gc_alloc_page_normal() == NULL) { @@ -472,13 +466,6 @@ gc_debug_verify_pool("pop before"); page->free_blocks--; SET_BIT1(page->block_bmp, GC_BLOCK_ID(block)); gc_pool_count--; -/*printf("POP pool count: %d block: %p page: %p sen: %p %p %p %p pool: %p\n", - gc_pool_count, block, page, - ¤t_thread->sentinels[0], - ¤t_thread->sentinels[1], - ¤t_thread->sentinels[2], - ¤t_thread->sentinels[3], - gc_pool); fflush(stdout);*/ hl_mutex_release(gc_mutex_pool); block->kind = GC_BLOCK_BRAND_NEW; block->owner_thread = hl_thread_id(); @@ -505,7 +492,6 @@ GC_STATIC void gc_push_block(gc_block_header_t *block) { // TODO: manage disorder in GC pool (prefer address order) SLL_PUSH(block, gc_pool); gc_pool_count++; -//printf("PUSH pool count: %d block: %p (%d) page: %p sen: %p pool: %p\n", gc_pool_count, block, block->kind, page, current_thread ? current_thread->sentinels : NULL, gc_pool); fflush(stdout); gc_stats->live_blocks--; hl_mutex_release(gc_mutex_pool); gc_debug_verify_pool("push"); @@ -543,8 +529,6 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) GC_ASSERT(words > 0); GC_ASSERT(size > 0); -// printf("pre alloc type: %p size: %d words: %d flags: %d start: %p\n", t, size, words, flags, current_thread->lines_start); - gc_object_t *ret = (gc_object_t *)current_thread->lines_start; gc_block_header_t *block = GC_LINE_BLOCK(ret); gc_metadata_t *meta = GC_METADATA(ret); @@ -552,6 +536,7 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) // clear stale data memset(ret, 0, sizeof(void *) * words); + // initialise type and metadata ret->t = t; meta->flags = 0; meta->marked = !gc_mark_polarity; @@ -563,25 +548,18 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) if (flags & MEM_KIND_NOPTR) { meta->no_ptr = 1; } - // int line_id = GC_LINE_ID(ret); - // gc_block_header_t *block = GC_LINE_BLOCK(ret); - // GC_DEBUG("pre-allocation: %p - %p", current_thread->lines_start, current_thread->lines_limit); - current_thread->lines_start = (void *)(((char *)current_thread->lines_start) + size); - if (LIKELY(size <= GC_LINE_SIZE)) { - // GC_DEBUG("allocated small %p in block %p (%d bytes, %d words)", ret, block, size, words); - } else { + if (UNLIKELY(size > GC_LINE_SIZE)) { meta->medium_sized = 1; GC_METADATA_EXT(ret) = sub_words >> 4; - // int last_id = GC_LINE_ID((void *)ret + size); - // ret->line_span = (last_id - line_id) + 1; - // GC_DEBUG("allocated medium %p in block %p (%d bytes, %d lines, %d words)", ret, block, size, ret->line_span, words); } - if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { // TODO: also in a finaliser-specific alloc function (after alloc_gen call) block->line_finalize[GC_LINE_ID(ret)] = 1; } + // bump cursor + current_thread->lines_start = (void *)(((char *)current_thread->lines_start) + size); + GC_DEBUG_DUMP3("gc_alloc_bump.success", ret, size, flags); gc_stats->live_memory += size; gc_stats->live_memory_normal += size; @@ -595,18 +573,6 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) // } SET_BIT1(block->debug_objects, obj_id); -// if (do_log) -// printf("postalloc: %p (%d bytes, %d flags)\n", ret, size, flags); - -//gc_page_header_t *page = GC_BLOCK_PAGE(block); -//if (ret == ID_TRACK(9, 0x2b5c80)) { -//if (ret >= ID_TRACK(12, 0x188900) && ret < ID_TRACK(12, 0x188980)) { -// gc_debug_block(block); -// printf("tracked alloc: %p (%d bytes, %d flags)\n", ret, size, flags); -// gc_debug_obj(ret); -// gc_debug_interactive(); -//} - return ret; } @@ -721,9 +687,10 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { return gc_alloc_bump(t, size, words, flags); } else { // TODO: separate path for huge objects -if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { - GC_FATAL("cannot alloc huge finalizer"); -} + if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { + // TODO: huge finalizer objects + GC_FATAL("cannot alloc huge finalizer"); + } gc_object_t *obj = gc_pop_huge(size); obj->t = t; gc_metadata_t *meta = GC_METADATA(obj); @@ -752,8 +719,8 @@ GC_STATIC void **get_stack_bottom(void) { // spills registers and triggers a major GC collection HL_API void hl_gc_major(void) { puts("major cycle..."); + // TODO: figure this out for the MT case (see master branch) //gc_stop_world(true); - //* jmp_buf env; setjmp(env); #pragma clang diagnostic push @@ -761,16 +728,6 @@ HL_API void hl_gc_major(void) { void *(*volatile f)(void) = get_stack_bottom; #pragma clang diagnostic pop current_thread->stack_cur = f(); - /* - printf("SPILLED REGISTERS:\n"); - void **a = &env; - void **b = (void **)((char *)a + sizeof(jmp_buf)); - for (; a < b; a++) { - printf("reg: %p\n", *a); - } - printf("marking thread from %p to %p\n", current_thread->stack_cur, current_thread->stack_top); - */ - //*/ GC_DEBUG_DUMP0("hl_gc_major.entry"); gc_mark(); gc_sweep(); @@ -816,7 +773,6 @@ GC_STATIC void gc_push(void *p, void *ref) { gc_mark_total++; gc_mark_stack_active->data[gc_mark_stack_active->pos].object = p; gc_mark_stack_active->data[gc_mark_stack_active->pos++].reference = ref; -//gc_page_header_t *page = GC_BLOCK_PAGE(p); GC_DEBUG(mark, "pushed %p -> %p to %p", ref, p, gc_mark_stack_active->data); GC_DEBUG_DUMP2("gc_push.success", p, ref); } @@ -844,8 +800,6 @@ GC_STATIC gc_object_t *gc_pop(void) { HL_PRIM vbyte* hl_type_name( hl_type *t ); -static unsigned long debug_scan_ctr = 0; - // marks live objects in the heap GC_STATIC void gc_mark(void) { GC_DEBUG(mark, "start"); @@ -891,20 +845,9 @@ GC_STATIC void gc_mark(void) { void **cur = t->stack_cur; void **top = t->stack_top; while (cur < top) { -//printf("stack: %p: %p\n", cur, *cur); - if (*cur == ID_TRACK(9, 0x34ab10)) { - gc_debug_obj_id(*cur); - printf("tracked push %p -> %p\n", cur, *cur); - } gc_push(*cur, NULL); cur++; - }/* - cur = (void **)(&(t->gc_regs)); - top = (void **)(((char *)&(t->gc_regs)) + sizeof(jmp_buf)); - while (cur < top) { - printf("reg: %p\n", *cur); - gc_push(*cur++, NULL); - }*/ + } } GC_DEBUG_DUMP0("gc_mark.propagate"); @@ -913,37 +856,22 @@ GC_STATIC void gc_mark(void) { while (gc_mark_total-- > 0) { gc_stats->live_objects++; gc_object_t *p = gc_pop(); -//if (debug_scan_ctr == 84) -// gc_debug_obj(p); -//printf("live object: %p %lu\n", p, debug_scan_ctr++); - if (p == NULL) - GC_FATAL("popped null"); gc_metadata_t *meta = GC_METADATA(p); gc_block_header_t *block = GC_LINE_BLOCK(p); -//if (p == ID_TRACK(9, 0x1af08)) -// puts("popped lock"); - // mark line(s) int line_id = GC_LINE_ID_IN(p, block); int obj_id = ((int_val)p - (int_val)&block->lines[0]) / 8; int words = meta->words; - /* - if (words == 0) { - if (GC_BLOCK_PAGE(p)->kind != GC_PAGE_HUGE && !GET_BIT(block->debug_objects, obj_id)) { - puts("non-ex!"); - } - gc_debug_obj(p); - GC_FATAL("popped 0-word object"); - } - */ - if (GC_BLOCK_PAGE(p)->kind != GC_PAGE_HUGE && !GET_BIT(block->debug_objects, obj_id)) { //gc_debug_block(block); // printf("popped: %p\n", p); fflush(stdout); //gc_debug_obj(p); //GC_FATAL("not an existing object!"); + // TODO: these objects probably come from a gc_major triggered from + // within gc_pop_block (or similar); in theory, interior pointers should + // not get here - then this check and debug_objects can be removed printf("non-ex %p\n", p); continue; } @@ -1027,10 +955,6 @@ GC_STATIC void gc_sweep(void) { for (int block_id = 0; block_id < GC_BLOCKS_PER_PAGE; block_id++) { if (GET_BIT(page->block_bmp, block_id)) { gc_block_header_t *block = GC_PAGE_BLOCK(page, block_id); - //if (block == GC_LINE_BLOCK(ID_TRACK(12, 0x188900))) { - // puts("DEBUG BLOCK"); - // gc_debug_obj(ID_TRACK(12, 0x188900)); - //} int lines_used = (block->line_marks[0] ? 1 : 0); for (int line_id = 0; line_id < GC_LINES_PER_BLOCK; line_id++) { gc_object_t *p = (gc_object_t *)&block->lines[line_id]; @@ -1038,31 +962,11 @@ GC_STATIC void gc_sweep(void) { if (!block->line_marks[line_id] && block->line_finalize[line_id]) { void **f = p; void (*finalize)(void *) = *((void (**)(void *))p); -/*if (page == tracked_page && ((int_val)p & 0x3FFFFF) == 0x2d2700) { - printf("before finalized!\n"); - gc_debug_obj(p); -}*/ -//if (p == ID_TRACK(9, 0x2b5c80)) { -// puts("DEBUG ABOUT TO FINALISE"); -// gc_debug_obj(p); -// printf("finalising block: %p, line: %d, obj: %p (page: %p, %ld), f: %p ... "/*" (*: %p) ... "*/, block, line_id, p, page, (int_val)p & 0x3FFFFF, finalize); //, finalize != NULL ? *(void **)(*f) : NULL); -// fflush(stdout); -//// gc_debug_interactive(); -//} -/*if (*(void **)(*f) == (void *)0x0B) { - gc_debug_obj(&block->lines[line_id]); - GC_FATAL("here!"); -} -fflush(stdout);*/ if (finalize != NULL) { finalize((void *)&block->lines[line_id]); *f = NULL; } block->line_finalize[line_id] = 0; -//if (p == ID_TRACK(9, 0x2b5c80)) { -// puts("ok"); -// fflush(stdout); -//} } if (!block->line_marks[line_id] && block->debug_objects) { for (int i = 0; i < 16; i++) { @@ -1101,7 +1005,6 @@ fflush(stdout);*/ gc_metadata_t *meta = GC_METADATA(obj); GC_DEBUG(sweep, "huge %p mark: %d %d", obj, meta->marked, gc_mark_polarity); if (meta->marked != gc_mark_polarity) { - //gc_free_page_memory(page); gc_push_huge(obj); } } @@ -1151,8 +1054,6 @@ HL_API bool hl_is_gc_ptr(void *ptr) { return (gc_get_block(ptr) != NULL); } -// HL_API varray *hl_alloc_array( hl_type *t, int size ) { return NULL; } - HL_API vdynamic *hl_alloc_dynamic( hl_type *t ) { return (vdynamic*)hl_gc_alloc_gen(t, sizeof(vdynamic), hl_is_ptr(t) ? MEM_KIND_DYNAMIC : MEM_KIND_NOPTR); } @@ -1184,8 +1085,6 @@ HL_API vdynamic *hl_alloc_obj( hl_type *t ) { return (vdynamic*)o; } -// HL_API venum *hl_alloc_enum( hl_type *t, int index ) { return NULL; } - HL_API vvirtual *hl_alloc_virtual( hl_type *t ) { vvirtual *v = (vvirtual*)hl_gc_alloc(t, t->virt->dataSize + sizeof(vvirtual) + sizeof(void*) * t->virt->nfields); void **fields = (void**)(v + 1); @@ -1203,7 +1102,10 @@ HL_API vdynobj *hl_alloc_dynobj(void) { return (vdynobj*)hl_gc_alloc_gen(&hlt_dynobj,sizeof(vdynobj), MEM_KIND_DYNAMIC); } -// HL_API vbyte *hl_alloc_bytes( int size ) { return NULL; } +// alloc functions defined elsewhere: +// varray *hl_alloc_array(hl_type *t, int size); +// venum *hl_alloc_enum(hl_type *t, int index); +// vbyte *hl_alloc_bytes(int size); static hl_types_dump gc_types_dump = NULL; HL_API void hl_gc_set_dump_types( hl_types_dump tdump ) { @@ -1320,6 +1222,8 @@ void hl_global_free() { hl_cache_free(); } +#if GC_ENABLE_DEBUG + void gc_debug_block(gc_block_header_t *block) { static char *kind_names[] = { "(not initialised)", @@ -1334,7 +1238,6 @@ void gc_debug_block(gc_block_header_t *block) { puts("block: NULL"); return; } - // printf("block: %p, kind: %s, prev: %p, next: %p\n", block, kind_names[block->kind], block->prev, block->next); printf("block: %p\n", block); gc_page_header_t *page = (gc_page_header_t *)block; printf(" (P) next_page: %p\n", page->next_page); @@ -1542,11 +1445,6 @@ void gc_debug_verify_pool(const char *at) { void *gc_debug_track_id(int pagenum, int low) { return ID_TRACK(pagenum, low); } -/* -void gc_debug_enable(bool dl) { - do_log = dl; -} -*/ void gc_debug_thread(void) { hl_thread_info *t = current_thread; @@ -1621,3 +1519,5 @@ void gc_debug_interactive(void) { } } } + +#endif // GC_ENABLE_DEBUG diff --git a/src/gc.h b/src/gc.h index 82465367e..9ebe763eb 100644 --- a/src/gc.h +++ b/src/gc.h @@ -263,30 +263,19 @@ GC_STATIC gc_block_header_t *gc_get_block(void *); // NULL if not a GC pointer # define GC_DEBUG_DUMP1(id, arg1) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); } while (0) # define GC_DEBUG_DUMP2(id, arg1, arg2) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1) + sizeof(arg2); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); GC_DEBUG_DUMP_R(arg2); } while (0) # define GC_DEBUG_DUMP3(id, arg1, arg2, arg3) do { GC_DEBUG_DUMP_S(id); int s = sizeof(arg1) + sizeof(arg2) + sizeof(arg3); GC_DEBUG_DUMP_R(s); GC_DEBUG_DUMP_R(arg1); GC_DEBUG_DUMP_R(arg2); GC_DEBUG_DUMP_R(arg3); } while (0) -#else -# define GC_DEBUG(...) -# define GC_DEBUG_DUMP_S(...) -# define GC_DEBUG_DUMP_R(...) -# define GC_DEBUG_DUMP0(...) -# define GC_DEBUG_DUMP1(...) -# define GC_DEBUG_DUMP2(...) -# define GC_DEBUG_DUMP3(...) -#endif -#define GC_FATAL(msg) do { puts(msg); gc_debug_interactive(); __builtin_trap(); } while (0) -#define GC_ASSERT(cond) do { \ +# define GC_ASSERT(cond) do { \ if (!(cond)) { \ printf("\x1B[38;5;160mGC assertion failed (line %d): " #cond "\n\x1B[0m", __LINE__); \ GC_FATAL("!"); \ } \ } while (0) -#define GC_ASSERT_M(cond, msg, ...) do { \ +# define GC_ASSERT_M(cond, msg, ...) do { \ if (!(cond)) { \ printf("\x1B[38;5;160mGC assertion failed (line %d): " #cond "\n\x1B[0m", __LINE__); \ printf("info: " msg "\n", ##__VA_ARGS__); \ GC_FATAL("!"); \ } \ } while (0) - GC_STATIC void gc_debug_block(gc_block_header_t *block); GC_STATIC void gc_debug_obj(void **p); GC_STATIC void gc_debug_obj_id(void **p); @@ -294,7 +283,26 @@ GC_STATIC void gc_debug_type(hl_type *t); GC_STATIC void gc_debug_interactive(void); GC_STATIC void gc_debug_verify_pool(const char *); GC_STATIC void *gc_debug_track_id(int, int); -//GC_STATIC void gc_debug_enable(bool); +#else +# define GC_DEBUG(...) +# define GC_DEBUG_DUMP_S(...) +# define GC_DEBUG_DUMP_R(...) +# define GC_DEBUG_DUMP0(...) +# define GC_DEBUG_DUMP1(...) +# define GC_DEBUG_DUMP2(...) +# define GC_DEBUG_DUMP3(...) +# define GC_ASSERT(...) +# define GC_ASSERT_M(...) +# define gc_debug_block(...) +# define gc_debug_obj(...) +# define gc_debug_obj_id(...) +# define gc_debug_type(...) +# define gc_debug_interactive(...) +# define gc_debug_verify_pool(...) +# define gc_debug_track_id(...) +#endif +#define GC_FATAL(msg) do { puts(msg); gc_debug_interactive(); __builtin_trap(); } while (0) + /* GC_STATIC void gc_debug_general(void); GC_STATIC void gc_dump(const char *); From 6643bb8f8f30a02bd36c4855256ee2f52a8e3c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Tue, 28 Apr 2020 15:21:19 +0100 Subject: [PATCH 07/13] typo --- src/hl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hl.h b/src/hl.h index 4a01fc2c7..1f291e54b 100644 --- a/src/hl.h +++ b/src/hl.h @@ -688,7 +688,7 @@ HL_API void hl_tls_free( hl_tls *l ); // ----------------------- GC ALLOC ----------------------------------------------- -#define MEM_HAS_PTR (!((kind)&2)) +#define MEM_HAS_PTR(kind) (!((kind)&2)) #define MEM_KIND_DYNAMIC 0 #define MEM_KIND_RAW 1 #define MEM_KIND_NOPTR 2 From de019b284137441a1cee54e46b9bace03a865916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Mon, 4 May 2020 21:41:41 +0100 Subject: [PATCH 08/13] fix huge allocations, fix stack top on GCC --- src/gc.c | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/src/gc.c b/src/gc.c index 3685ed84f..61629771b 100644 --- a/src/gc.c +++ b/src/gc.c @@ -129,6 +129,8 @@ GC_STATIC void *gc_alloc_os_memory(int size) { } gc_stats->total_memory += size; base_addr = (char *)ptr + size; + // align up + base_addr = (void *)((((int_val)base_addr) & ~(GC_PAGE_SIZE - 1)) + GC_PAGE_SIZE); //printf("PAGE ALLOC %d: %p (%d bytes)\n", page_counter++, ptr, size); GC_DEBUG_DUMP2("gc_alloc_os_memory.success", ptr, size); GC_DEBUG(os, "allocated OS page: %p", ptr); @@ -708,13 +710,16 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { } } -GC_STATIC void **get_stack_bottom(void) { - void *x; -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wreturn-stack-address" +#ifdef __clang__ +// this only works with clang +static void *get_stack_bottom(void) { + int x; + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wreturn-stack-address" return &x; -#pragma clang diagnostic pop + #pragma clang diagnostic pop } +#endif // spills registers and triggers a major GC collection HL_API void hl_gc_major(void) { @@ -723,11 +728,15 @@ HL_API void hl_gc_major(void) { //gc_stop_world(true); jmp_buf env; setjmp(env); -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wincompatible-pointer-types" +#ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wincompatible-pointer-types" void *(*volatile f)(void) = get_stack_bottom; -#pragma clang diagnostic pop + #pragma clang diagnostic pop current_thread->stack_cur = f(); +#else + current_thread->stack_cur = __builtin_frame_address(0); +#endif GC_DEBUG_DUMP0("hl_gc_major.entry"); gc_mark(); gc_sweep(); @@ -1124,6 +1133,7 @@ GC_STATIC void gc_handle_signal(int signum) { } GC_STATIC gc_stats_t *gc_init(void) { + puts("new GC init..."); gc_mutex_roots = hl_mutex_alloc(false); gc_mutex_pool = hl_mutex_alloc(false); gc_blocked = false; @@ -1217,7 +1227,6 @@ void hl_global_init() { } void hl_global_free() { - printf("%lu\n", gc_stats->cycles); gc_deinit(); hl_cache_free(); } @@ -1521,3 +1530,22 @@ void gc_debug_interactive(void) { } #endif // GC_ENABLE_DEBUG + +// TODO +void hl_gc_enable(bool a) {} +void hl_gc_profile(bool a) {} +void hl_gc_stats(double *a, double *b, double *c) {} +void hl_gc_dump_memory(vbyte *a) {} +int hl_gc_get_flags(void) { return 0; } +void hl_gc_set_flags(int a) {} +vdynamic *hl_debug_call(int a, vdynamic *b) { return NULL; } + +DEFINE_PRIM(_VOID, gc_major, _NO_ARG); +DEFINE_PRIM(_VOID, gc_enable, _BOOL); +DEFINE_PRIM(_VOID, gc_profile, _BOOL); +DEFINE_PRIM(_VOID, gc_stats, _REF(_F64) _REF(_F64) _REF(_F64)); +DEFINE_PRIM(_VOID, gc_dump_memory, _BYTES); +DEFINE_PRIM(_I32, gc_get_flags, _NO_ARG); +DEFINE_PRIM(_VOID, gc_set_flags, _I32); +DEFINE_PRIM(_DYN, debug_call, _I32 _DYN); +DEFINE_PRIM(_VOID, blocking, _BOOL); From 4140a08881162f0d65ba88112defb239797cc88b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Tue, 5 May 2020 22:11:13 +0100 Subject: [PATCH 09/13] fix a crash (somehow) --- src/gc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gc.c b/src/gc.c index 61629771b..f83a6f092 100644 --- a/src/gc.c +++ b/src/gc.c @@ -712,8 +712,8 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { #ifdef __clang__ // this only works with clang -static void *get_stack_bottom(void) { - int x; +static void **get_stack_bottom(void) { + void *x; #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wreturn-stack-address" return &x; From a889870524ed009974ae1cfb605788e4c486d323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Thu, 21 May 2020 20:15:35 +0100 Subject: [PATCH 10/13] fix huge object allocation --- src/gc.c | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/src/gc.c b/src/gc.c index f83a6f092..4c1225515 100644 --- a/src/gc.c +++ b/src/gc.c @@ -639,6 +639,29 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { size = words * sizeof(void *); + // never allocate huge objects inside blocks + if (size > GC_MEDIUM_SIZE) { + // TODO: (statically) separate path for huge objects + if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { + // TODO: huge finalizer objects + GC_FATAL("cannot alloc huge finalizer"); + } + gc_object_t *obj = gc_pop_huge(size); + obj->t = t; + gc_metadata_t *meta = GC_METADATA(obj); + meta->flags = 0; + meta->marked = !gc_mark_polarity; + if (flags & MEM_KIND_RAW) { + meta->raw = 1; + } + if (flags & MEM_KIND_NOPTR) { + meta->no_ptr = 1; + } + gc_stats->live_memory += size; + dump_live(); + return obj; + } + if (LIKELY((char *)current_thread->lines_start + size <= (char *)current_thread->lines_limit)) { return gc_alloc_bump(t, size, words, flags); } @@ -677,7 +700,7 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { current_thread->lines_start = ¤t_thread->lines_block->lines[0]; current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; return gc_alloc_bump(t, size, words, flags); - } else if (size <= GC_MEDIUM_SIZE) { + } else { // size <= GC_MEDIUM_SIZE current_thread->lines_block->kind = GC_BLOCK_FULL; DLL_INSERT(current_thread->lines_block, current_thread->full_blocks); gc_debug_verify_pool("full insert medium"); @@ -687,26 +710,6 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { current_thread->lines_start = ¤t_thread->lines_block->lines[0]; current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; return gc_alloc_bump(t, size, words, flags); - } else { - // TODO: separate path for huge objects - if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { - // TODO: huge finalizer objects - GC_FATAL("cannot alloc huge finalizer"); - } - gc_object_t *obj = gc_pop_huge(size); - obj->t = t; - gc_metadata_t *meta = GC_METADATA(obj); - meta->flags = 0; - meta->marked = !gc_mark_polarity; - if (flags & MEM_KIND_RAW) { - meta->raw = 1; - } - if (flags & MEM_KIND_NOPTR) { - meta->no_ptr = 1; - } - gc_stats->live_memory += size; - dump_live(); - return obj; } } From 57bad33a500dcbacdddb5166716d8f4f6d58f4ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Fri, 22 May 2020 14:05:10 +0100 Subject: [PATCH 11/13] turn off memory limiting --- src/gc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gc.c b/src/gc.c index 4c1225515..907f31d60 100644 --- a/src/gc.c +++ b/src/gc.c @@ -97,11 +97,13 @@ static FILE *dump_file; // allocates a page-aligned region of memory from the OS GC_STATIC void *gc_alloc_os_memory(int size) { GC_DEBUG_DUMP1("gc_alloc_os_memory.enter", size); + /* if (gc_stats->total_memory + size >= gc_config->memory_limit) { GC_DEBUG_DUMP0("gc_alloc_os_memory.fail.oom"); GC_DEBUG(fatal, "using %lu / %lu bytes, need %d more", gc_stats->total_memory, gc_config->memory_limit, size); GC_FATAL("OOM: memory limit hit"); } + */ void *ptr = mmap(base_addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (ptr == (void *)-1) { GC_DEBUG_DUMP0("gc_alloc_os_memory.fail.other"); From 7c66c599d03d61e3de9e571f3af3621f23379009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Sat, 23 May 2020 12:17:37 +0100 Subject: [PATCH 12/13] allow huge object storage in blocks, separate path for finaliser alloc --- src/gc.c | 74 ++++++++++++++++++++++++++++---------------------------- src/gc.h | 2 +- src/hl.h | 2 +- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/src/gc.c b/src/gc.c index 907f31d60..1d9821e77 100644 --- a/src/gc.c +++ b/src/gc.c @@ -556,10 +556,6 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) meta->medium_sized = 1; GC_METADATA_EXT(ret) = sub_words >> 4; } - if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { - // TODO: also in a finaliser-specific alloc function (after alloc_gen call) - block->line_finalize[GC_LINE_ID(ret)] = 1; - } // bump cursor current_thread->lines_start = (void *)(((char *)current_thread->lines_start) + size); @@ -630,40 +626,8 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { // align to words int words = (size + (sizeof(void *) - 1)) / sizeof(void *); - if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { - // TODO: separate function to do this before alloc_gen - // (to reduce hot path for non-finaliser objects) - - // align to cache lines - words = (((size + 127) / 128) * 128) / sizeof(void *); - current_thread->lines_start = (int_val)((char *)current_thread->lines_start + 127) & 0xFFFFFFFFFFFFFF80; - } - size = words * sizeof(void *); - // never allocate huge objects inside blocks - if (size > GC_MEDIUM_SIZE) { - // TODO: (statically) separate path for huge objects - if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { - // TODO: huge finalizer objects - GC_FATAL("cannot alloc huge finalizer"); - } - gc_object_t *obj = gc_pop_huge(size); - obj->t = t; - gc_metadata_t *meta = GC_METADATA(obj); - meta->flags = 0; - meta->marked = !gc_mark_polarity; - if (flags & MEM_KIND_RAW) { - meta->raw = 1; - } - if (flags & MEM_KIND_NOPTR) { - meta->no_ptr = 1; - } - gc_stats->live_memory += size; - dump_live(); - return obj; - } - if (LIKELY((char *)current_thread->lines_start + size <= (char *)current_thread->lines_limit)) { return gc_alloc_bump(t, size, words, flags); } @@ -702,7 +666,7 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { current_thread->lines_start = ¤t_thread->lines_block->lines[0]; current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; return gc_alloc_bump(t, size, words, flags); - } else { // size <= GC_MEDIUM_SIZE + } else if (size <= GC_MEDIUM_SIZE) { current_thread->lines_block->kind = GC_BLOCK_FULL; DLL_INSERT(current_thread->lines_block, current_thread->full_blocks); gc_debug_verify_pool("full insert medium"); @@ -712,9 +676,45 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { current_thread->lines_start = ¤t_thread->lines_block->lines[0]; current_thread->lines_limit = ¤t_thread->lines_block->lines[GC_LINES_PER_BLOCK]; return gc_alloc_bump(t, size, words, flags); + } else { + // TODO: (statically) separate path for huge objects + if ((flags & MEM_KIND_FINALIZER) == MEM_KIND_FINALIZER) { + // TODO: huge finalizer objects + GC_FATAL("cannot alloc huge finalizer"); + } + gc_object_t *obj = gc_pop_huge(size); + obj->t = t; + gc_metadata_t *meta = GC_METADATA(obj); + meta->flags = 0; + meta->marked = !gc_mark_polarity; + if (flags & MEM_KIND_RAW) { + meta->raw = 1; + } + if (flags & MEM_KIND_NOPTR) { + meta->no_ptr = 1; + } + gc_stats->live_memory += size; + dump_live(); + return obj; } } +HL_API void *hl_gc_alloc_finalizer(int size) { + // align to cache lines + int words = (size + (sizeof(void *) - 1)) / sizeof(void *); + words = (((size + 127) / 128) * 128) / sizeof(void *); + current_thread->lines_start = (int_val)((char *)current_thread->lines_start + 127) & 0xFFFFFFFFFFFFFF80; + size = words * sizeof(void *); + + void *ret = hl_gc_alloc_gen(&hlt_abstract, size, MEM_KIND_FINALIZER); + + // mark as finaliser + gc_block_header_t *block = GC_LINE_BLOCK(ret); + block->line_finalize[GC_LINE_ID(ret)] = 1; + + return ret; +} + #ifdef __clang__ // this only works with clang static void **get_stack_bottom(void) { diff --git a/src/gc.h b/src/gc.h index 9ebe763eb..7e6c5f806 100644 --- a/src/gc.h +++ b/src/gc.h @@ -220,7 +220,7 @@ GC_STATIC void gc_grow_heap(int count); #define GC_LINE_ID(ptr) GC_LINE_ID_IN(ptr, GC_LINE_BLOCK(ptr)) #define GC_METADATA(obj) (&(GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024])) -#define GC_METADATA_EXT(obj) *(unsigned char *)(&GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024 + 1]) +#define GC_METADATA_EXT(obj) *(int *)(&GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024 + 1]) // roots ----------------------------------------------------------- diff --git a/src/hl.h b/src/hl.h index 1f291e54b..ccecf9c4b 100644 --- a/src/hl.h +++ b/src/hl.h @@ -697,6 +697,7 @@ HL_API void hl_tls_free( hl_tls *l ); #define MEM_ZERO 256 HL_API void *hl_gc_alloc_gen( hl_type *t, int size, int flags ); +HL_API void *hl_gc_alloc_finalizer( int size ); HL_API void hl_add_root( void **ptr ); HL_API void hl_remove_root( void **ptr ); HL_API void hl_gc_major( void ); @@ -711,7 +712,6 @@ HL_API void hl_gc_set_dump_types( hl_types_dump tdump ); #define hl_gc_alloc_noptr(size) hl_gc_alloc_gen(&hlt_bytes, size, MEM_KIND_NOPTR) #define hl_gc_alloc(t, size) hl_gc_alloc_gen(t, size, MEM_KIND_DYNAMIC) #define hl_gc_alloc_raw(size) hl_gc_alloc_gen(&hlt_abstract, size, MEM_KIND_RAW) -#define hl_gc_alloc_finalizer(size) hl_gc_alloc_gen(&hlt_abstract, size, MEM_KIND_FINALIZER) // ----------------------- INTERNAL ALLOC ----------------------------------------- From 75d230c7f6e6d9be9cac66c79b59cab30b4567ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurel=20Bi=CC=81ly=CC=81?= Date: Mon, 25 May 2020 10:37:01 +0100 Subject: [PATCH 13/13] embed object bmp into block header, bugfixes --- other/gctests/bench/mandelbrot.c | 2 +- other/gctests/main.c | 29 +++++++++++++-------- src/gc.c | 43 +++++++++++++++++++------------- src/gc.h | 19 ++++++++------ 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/other/gctests/bench/mandelbrot.c b/other/gctests/bench/mandelbrot.c index 48ed3f7a5..11ace2a9d 100644 --- a/other/gctests/bench/mandelbrot.c +++ b/other/gctests/bench/mandelbrot.c @@ -37,7 +37,7 @@ TEST_TYPE(rgb, 3, {0}, { int g; int b; }); -TEST_TYPE(complex, 3, {0}, { +TEST_TYPE(complex, 2, {0}, { double i; double j; }); diff --git a/other/gctests/main.c b/other/gctests/main.c index d34fc665e..2720ba074 100644 --- a/other/gctests/main.c +++ b/other/gctests/main.c @@ -46,9 +46,9 @@ BEGIN_TEST_CASE(sanity) { ASSERT(sizeof(hl_type *) == 8); ASSERT(sizeof(gc_page_header_t) == 32); ASSERT(sizeof(gc_metadata_t) == 1); - // ASSERT(sizeof(gc_metadata_ext_t) == 2); ASSERT(sizeof(gc_block_header_t) == GC_BLOCK_SIZE); - ASSERT(offsetof(gc_block_header_t, lines) == 64 * GC_LINE_SIZE); + ASSERT(offsetof(gc_block_header_t, metadata) == 64); + ASSERT(offsetof(gc_block_header_t, lines) == 70 * GC_LINE_SIZE); ASSERT(GC_PAGE_BLOCK(0x686178650B400000ul, 0) == (gc_block_header_t *)0x686178650B400000ul); ASSERT(GC_PAGE_BLOCK(0x686178650B400000ul, 13) == (gc_block_header_t *)0x686178650B4D0000ul); @@ -69,15 +69,22 @@ BEGIN_TEST_CASE(sanity) { ASSERT(GC_LINE_BLOCK(0x686178650B4D2123ul) == (gc_block_header_t *)0x686178650B4D0000ul); ASSERT(GC_LINE_BLOCK(0x686178650B4DFF80ul) == (gc_block_header_t *)0x686178650B4D0000ul); - ASSERT(GC_LINE_ID(0x686178650B4D2000ul) == 0); - ASSERT(GC_LINE_ID(0x686178650B4D2123ul) == 2); - ASSERT(GC_LINE_ID(0x686178650B4DFF80ul) == 447); + ASSERT(GC_LINE_ID(0x686178650B4D2300ul) == 0); + ASSERT(GC_LINE_ID(0x686178650B4D2423ul) == 2); + ASSERT(GC_LINE_ID(0x686178650B4DFF80ul) == 441); - ASSERT(GC_METADATA(0x686178650B4D2000ul) == (gc_metadata_t *)0x686178650B4D0200ul); - ASSERT(GC_METADATA(0x686178650B4D2010ul) == (gc_metadata_t *)0x686178650B4D0202ul); - ASSERT(GC_METADATA(0x686178650B4D2018ul) == (gc_metadata_t *)0x686178650B4D0203ul); - ASSERT(GC_METADATA(0x686178650B4D2128ul) == (gc_metadata_t *)0x686178650B4D0225ul); - ASSERT(GC_METADATA(0x686178650B4DFFF0ul) == (gc_metadata_t *)0x686178650B4D1DFEul); + ASSERT(GC_OBJ_ID(0x686178650B4D2300ul) == 0); + ASSERT(GC_OBJ_ID(0x686178650B4D2308ul) == 1); + ASSERT(GC_OBJ_ID(0x686178650B4D2310ul) == 2); + ASSERT(GC_OBJ_ID(0x686178650B4D2318ul) == 3); + ASSERT(GC_OBJ_ID(0x686178650B4D2428ul) == 37); + ASSERT(GC_OBJ_ID(0x686178650B4DFFF8ul) == 7071); + + ASSERT(GC_METADATA(0x686178650B4D2300ul) == (gc_metadata_t *)0x686178650B4D0040ul); + ASSERT(GC_METADATA(0x686178650B4D2310ul) == (gc_metadata_t *)0x686178650B4D0042ul); + ASSERT(GC_METADATA(0x686178650B4D2318ul) == (gc_metadata_t *)0x686178650B4D0043ul); + ASSERT(GC_METADATA(0x686178650B4D2428ul) == (gc_metadata_t *)0x686178650B4D0065ul); + ASSERT(GC_METADATA(0x686178650B4DFFF0ul) == (gc_metadata_t *)0x686178650B4D1BDEul); char cs[] = "\x00\x01\x02\x03\x04\x05"; ASSERT(*(int *)(&cs[0]) == 0x03020100); @@ -342,7 +349,7 @@ int main(int argc, char **argv) { RUN_TEST(big_object); RUN_TEST(simple_array); RUN_TEST(big_array); - //RUN_TEST(many_trees); + RUN_TEST(many_trees); RUN_TEST(finalizer); puts("---"); puts("TOTAL:"); diff --git a/src/gc.c b/src/gc.c index 1d9821e77..edf576729 100644 --- a/src/gc.c +++ b/src/gc.c @@ -217,9 +217,6 @@ if (page_counter < 500) for (int i = 0; i < GC_BLOCKS_PER_PAGE; i++) { blocks[i].kind = GC_BLOCK_FREE; blocks[i].next = &blocks[i + 1]; - blocks[i].debug_objects = (char *)calloc(1, 896); - if (blocks[i].debug_objects == NULL) - GC_FATAL("cannot alloc debug bitmap?"); } hl_mutex_acquire(gc_mutex_pool); blocks[GC_BLOCKS_PER_PAGE - 1].next = gc_pool; @@ -485,9 +482,15 @@ gc_debug_verify_pool("pop after"); GC_STATIC void gc_push_block(gc_block_header_t *block) { GC_ASSERT(block != NULL); GC_ASSERT(block->kind == GC_BLOCK_FULL); + + // clear old data block->kind = GC_BLOCK_FREE; // TODO: zombie ? block->owner_thread = -1; + memset(block->metadata, 0, GC_LINES_PER_BLOCK * 16); memset(block->line_marks, 0, GC_LINES_PER_BLOCK); + memset(block->line_finalize, 0, GC_LINES_PER_BLOCK); + memset(block->objects_bmp, 0, GC_LINES_PER_BLOCK * 2); + gc_page_header_t *page = GC_BLOCK_PAGE(block); // TODO: mutex per page? hl_mutex_acquire(gc_mutex_pool); @@ -505,7 +508,7 @@ gc_debug_verify_pool("push"); // acquires a huge object // this always allocates an OS page GC_STATIC void *gc_pop_huge(int size) { - gc_page_header_t *header = gc_alloc_page_huge(8192 + size); + gc_page_header_t *header = gc_alloc_page_huge((GC_BLOCK_HEADER_LINES * GC_LINE_SIZE) + size); GC_DEBUG(alloc, "allocated huge %d at %p", size, header); GC_DEBUG_DUMP2("gc_pop_huge.success", header, size); gc_block_header_t *block = (gc_block_header_t *)header; @@ -571,7 +574,7 @@ GC_STATIC gc_object_t *gc_alloc_bump(hl_type *t, int size, int words, int flags) //gc_debug_block(block); // GC_FATAL("overriding existing object"); // } - SET_BIT1(block->debug_objects, obj_id); + SET_BIT1(block->objects_bmp, obj_id); return ret; } @@ -608,7 +611,7 @@ GC_STATIC bool gc_find_gap(void) { // GC_DEBUG("gap search in %p; line %d, %p", current_thread->lines_block, line, current_thread->lines_start); // found a gap, increase limit as far as possible // TODO: optimise - while (current_thread->lines_block->line_marks[line] == 0) { + while (current_thread->lines_block->line_marks[line] == 0 && line < GC_LINES_PER_BLOCK) { line++; } current_thread->lines_limit = ¤t_thread->lines_block->lines[line]; @@ -633,7 +636,7 @@ HL_API void *hl_gc_alloc_gen(hl_type *t, int size, int flags) { } // align start to next cache line (for gc_find_gap) - current_thread->lines_start = (int_val)((char *)current_thread->lines_start + 127) & 0xFFFFFFFFFFFFFF80; + current_thread->lines_start = (int_val)((char *)current_thread->lines_start + GC_LINE_SIZE) & 0xFFFFFFFFFFFFFF80; // TODO: list operations should not be interrupted by the GC, mutex per list @@ -875,29 +878,29 @@ GC_STATIC void gc_mark(void) { // mark line(s) int line_id = GC_LINE_ID_IN(p, block); - int obj_id = ((int_val)p - (int_val)&block->lines[0]) / 8; + int obj_id = GC_OBJ_ID(p); int words = meta->words; - if (GC_BLOCK_PAGE(p)->kind != GC_PAGE_HUGE && !GET_BIT(block->debug_objects, obj_id)) { + if (GC_BLOCK_PAGE(p)->kind != GC_PAGE_HUGE && !GET_BIT(block->objects_bmp, obj_id)) { //gc_debug_block(block); // printf("popped: %p\n", p); fflush(stdout); //gc_debug_obj(p); //GC_FATAL("not an existing object!"); // TODO: these objects probably come from a gc_major triggered from // within gc_pop_block (or similar); in theory, interior pointers should - // not get here - then this check and debug_objects can be removed + // not get here - then this check and maybe objects_bmp ? can be removed printf("non-ex %p\n", p); continue; } if (block->huge_sized) { - words = (GC_BLOCK_PAGE(p)->size - 8192) / 8; + words = (GC_BLOCK_PAGE(p)->size - (GC_BLOCK_HEADER_LINES * GC_LINE_SIZE)) / 8; GC_DEBUG(mark, "scanning huge, %d words", words); } else if (meta->medium_sized) { words |= GC_METADATA_EXT(p) << 4; words++; - int last_id = GC_LINE_ID_IN((char *)p + words * 8, block); - int line_span = (last_id - line_id); + int last_id = GC_LINE_ID_IN((char *)p + words * 8 - 1, block); + int line_span = (last_id - line_id) + 1; GC_DEBUG(mark, "marking medium, %d lines", line_span); memset(&block->line_marks[line_id], 1, line_span); } else if (!block->huge_sized) { @@ -982,11 +985,10 @@ GC_STATIC void gc_sweep(void) { } block->line_finalize[line_id] = 0; } - if (!block->line_marks[line_id] && block->debug_objects) { - for (int i = 0; i < 16; i++) { - int obj_id = line_id * 16 + i; - SET_BIT0(block->debug_objects, obj_id); - } + if (!block->line_marks[line_id]) { + // clear all 16 bits + block->objects_bmp[line_id * 2] = 0; + block->objects_bmp[line_id * 2 + 1] = 0; } if (line_id == 0) continue; @@ -1049,6 +1051,11 @@ GC_STATIC gc_block_header_t *gc_get_block(void *p) { // block header return NULL; } + int obj_id = ((int_val)p - (int_val)&ret->lines[0]) / 8; + /*if (!GET_BIT(ret->objects_bmp, obj_id)) { + // not initialised + return NULL; + }*/ return ret; } } diff --git a/src/gc.h b/src/gc.h index 7e6c5f806..72b653c34 100644 --- a/src/gc.h +++ b/src/gc.h @@ -16,13 +16,14 @@ static void gc_free_os_memory(void *ptr, int size); // 4 MiB pages // 64 KiB blocks (64 blocks per page) -// 128 B lines (8KiB header + 448 lines per block) +// 128 B lines (8KiB header + 442 lines per block) #define GC_PAGE_SIZE (1 << 22) #define GC_BLOCK_SIZE (1 << 16) #define GC_LINE_SIZE (1 << 7) #define GC_BLOCKS_PER_PAGE 64 -#define GC_LINES_PER_BLOCK 448 +#define GC_LINES_PER_BLOCK 442 +#define GC_BLOCK_HEADER_LINES (512 - GC_LINES_PER_BLOCK) // up to 8184 (~25% of block size) for medium objects #define GC_MEDIUM_SIZE ((1 << 13) - 8) @@ -118,10 +119,10 @@ typedef struct gc_block_header_s { int owner_thread; int huge_sized; unsigned char _pad[4]; - unsigned char line_marks[GC_LINES_PER_BLOCK]; gc_metadata_t metadata[GC_LINES_PER_BLOCK * 16]; - char *debug_objects; + unsigned char line_marks[GC_LINES_PER_BLOCK]; unsigned char line_finalize[GC_LINES_PER_BLOCK]; + unsigned char objects_bmp[GC_LINES_PER_BLOCK * 2]; unsigned char _pad2[56]; gc_line_t lines[GC_LINES_PER_BLOCK]; } gc_block_header_t; @@ -132,6 +133,7 @@ typedef struct gc_block_dummy_s { gc_block_header_t *next; } gc_block_dummy_t; +/* typedef struct { // bool gc_blocking; void *lines_start; @@ -141,6 +143,7 @@ typedef struct { gc_block_header_t *full_blocks; // DLL of all full blocks, may become recycled gc_block_header_t *recyclable_blocks; // DLL of recyclable blocks } gc_thread_info_t; +*/ typedef struct { hl_type *t; @@ -215,12 +218,14 @@ GC_STATIC void gc_grow_heap(int count); // gets the index of `block` in its page #define GC_BLOCK_ID(block) ((int)(((int_val)(block) - (int_val)GC_BLOCK_PAGE(block)) / GC_BLOCK_SIZE)) // gets the line index of `ptr` in the given block -#define GC_LINE_ID_IN(ptr, block) ((int)((int_val)(ptr) - (int_val)(block)) / GC_LINE_SIZE - 64) +#define GC_LINE_ID_IN(ptr, block) ((int)((int_val)(ptr) - (int_val)(block)) / GC_LINE_SIZE - GC_BLOCK_HEADER_LINES) // gets the line index of `ptr` in its block #define GC_LINE_ID(ptr) GC_LINE_ID_IN(ptr, GC_LINE_BLOCK(ptr)) -#define GC_METADATA(obj) (&(GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024])) -#define GC_METADATA_EXT(obj) *(int *)(&GC_LINE_BLOCK(obj)->metadata[((int_val)(obj) - (int_val)GC_LINE_BLOCK(obj)) / 8 - 1024 + 1]) +#define GC_OBJ_ID(obj) ((int_val)(obj) - (int_val)(&GC_LINE_BLOCK(obj)->lines[0])) / 8 + +#define GC_METADATA(obj) (&(GC_LINE_BLOCK(obj)->metadata[GC_OBJ_ID(obj)])) +#define GC_METADATA_EXT(obj) *(int *)(&GC_LINE_BLOCK(obj)->metadata[GC_OBJ_ID(obj) + 1]) // roots -----------------------------------------------------------