From 801b4bc2c05fff0690ecfe4511a91e07e286602c Mon Sep 17 00:00:00 2001
From: Paul Guyot <pguyot@kallisys.net>
Date: Sun, 20 Aug 2023 11:23:27 +0200
Subject: [PATCH] Implement heap growth strategies

Strategies can be selected with `spawn_opt/2,4` and affect how heap is grown.
Three strategies are implemented:
- bounded free: keep free space within bounds (this is the old implementation)
- minimum: always try to mimimize free space
- fibonacci: grow heap following fibonacci up to a certain point
  (inspired from Erlang/OTP)

Also fix allocation with min_heap_size (a bug in strategy generated a lot of
useless garbace collections).
Also fix semantic of `heap_size` for process_info and add `total_heap_size`.

Signed-off-by: Paul Guyot <pguyot@kallisys.net>
---
 doc/src/memory-management.md              |  16 +-
 libs/estdlib/src/erlang.erl               |  10 +-
 src/libAtomVM/context.c                   |  12 +-
 src/libAtomVM/context.h                   |   8 +
 src/libAtomVM/defaultatoms.c              |  12 ++
 src/libAtomVM/defaultatoms.h              |  14 +-
 src/libAtomVM/memory.c                    | 110 +++++++++--
 src/libAtomVM/memory.h                    |  14 +-
 src/libAtomVM/nifs.c                      |  24 ++-
 tests/erlang_tests/CMakeLists.txt         |   2 +
 tests/erlang_tests/test_heap_growth.erl   | 216 ++++++++++++++++++++++
 tests/erlang_tests/test_min_heap_size.erl |  74 +++-----
 tests/erlang_tests/test_process_info.erl  |  10 +-
 tests/erlang_tests/test_refc_binaries.erl |   4 +-
 tests/erlang_tests/test_sub_binaries.erl  |   4 +-
 tests/test.c                              |   1 +
 16 files changed, 458 insertions(+), 73 deletions(-)
 create mode 100644 tests/erlang_tests/test_heap_growth.erl

diff --git a/doc/src/memory-management.md b/doc/src/memory-management.md
index 20ecd28cd..eb94e161a 100644
--- a/doc/src/memory-management.md
+++ b/doc/src/memory-management.md
@@ -26,7 +26,7 @@ The heap and stack for each AtomVM process are stored in a single allocated bloc
 
 The heap contains all of the allocated terms in an execution context.  In some cases, the terms occupy more than one word of memory (e.g., a tuple), but in general, the heap contains a record of memory in use by the program.
 
-The heap grows incrementally, as memory is allocated, and terms are allocated sequentially, in increasing memory addresses.  There is, therefore, no memory fragmentation, properly speaking, at least insofar as a portion of memory might be in use and then freed.  However, it is possible that previously allocated blocks of memory in the context heap are no longer referenced by the program.  In this case, the allocated blocks are "garbage", and are reclaimed at the next garbage collection.
+The heap grows incrementally, as memory is allocated, and terms are allocated sequentially, in increasing memory addresses.  There is, therefore, no memory fragmentation, properly speaking, at least insofar as a portion of memory might be in use and then freed.  However, it is possible that previously allocated blocks of memory in the context heap are no longer referenced by the program.  In this case, the allocated blocks are "garbage", and are reclaimed at the next garbage collection. The actual growth of the heap is controlled by a heap growth strategy (`heap_growth` spawn option) as described below.
 
 > Note. It is possible for the AtomVM heap, as provided by the underlying operating system, to become fragmented, as the execution context stack and heap are allocated via `malloc` or equiv.  But that is a different kind of fragmentation that does not refer to the allocated block used by an individual AtomVM process.
 
@@ -64,7 +64,19 @@ The following diagram illustrates an allocated block of memory that stores terms
     |           word[n-1]            |      v      v
     +================================+ <- stack_base --
 
-The initial size of the allocated block for the stack and heap in AtomVM is 8 words.  As heap and stack allocations grow, eventually, the amount of free space will decrease to the point where a garbage collection is required.  In this case, a new but larger (typically by 2x) block of memory is allocated by the AtomVM OS process, and terms are copied from the old stack and heap to the new stack and heap.  Garbage collection is described in more detail below.
+The initial size of the allocated block for the stack and heap in AtomVM is 8 words.  As heap and stack allocations grow, eventually, the amount of free space will decrease to the point where a garbage collection is required.  In this case, a new but larger block of memory is allocated by the AtomVM OS process, and terms are copied from the old stack and heap to the new stack and heap.  Garbage collection is described in more detail below.
+
+### Heap growth strategies
+
+AtomVM aims at minimizing memory footprint and several heap growth strategies are available. The heap is grown or shrunk when an allocation is required and the current execution context allows for a garbage collection (that will move data structures), allows for shrinking or forces shrinking (typically in the case of a call to `erlang:garbage_collect/0,1`).
+
+Each strategy is set at the process level.
+
+Default strategy is bounded free (`{heap_growth, bounded_free}`). In this strategy, when more memory is required, the allocator keeps the free amount between fixed boundaries (currently 16 and 32 terms). If no allocation is required but free space is larger than boundary, a garbage collection is triggered. After copying data to a new heap, if the free space is larger than the maximum, the heap is shrunk within the boundaries.
+
+With minimum strategy (`{heap_growth, minimum}`), when an allocation can happen, it is always adjusted to have the free space at 0.
+
+With fibonacci strategy (`{heap_growth, fibonacci}`), heap size grows following a variation of fibonacci until a large value and then grows by 20%. If free space is larger than 75% of heap size, the heap is shrunk. This strategy is inspired from Erlang/OTP's implementation.
 
 ### Registers
 
diff --git a/libs/estdlib/src/erlang.erl b/libs/estdlib/src/erlang.erl
index fe180515c..e4f4e33d2 100644
--- a/libs/estdlib/src/erlang.erl
+++ b/libs/estdlib/src/erlang.erl
@@ -193,7 +193,8 @@ send_after(Time, Dest, Msg) ->
 %%
 %% The following keys are supported:
 %% <ul>
-%%      <li><b>heap_size</b> the number of words used in the heap (integer)</li>
+%%      <li><b>heap_size</b> the number of words used in the heap (integer), including the stack but excluding fragments</li>
+%%      <li><b>total_heap_size</b> the number of words used in the heap (integer) including fragments</li>
 %%      <li><b>stack_size</b> the number of words used in the stack (integer)</li>
 %%      <li><b>message_queue_len</b> the number of messages enqueued for the process (integer)</li>
 %%      <li><b>memory</b> the estimated total number of bytes in use by the process (integer)</li>
@@ -205,6 +206,7 @@ send_after(Time, Dest, Msg) ->
 %%-----------------------------------------------------------------------------
 -spec process_info
     (Pid :: pid(), heap_size) -> {heap_size, non_neg_integer()};
+    (Pid :: pid(), total_heap_size) -> {total_heap_size, non_neg_integer()};
     (Pid :: pid(), stack_size) -> {stack_size, non_neg_integer()};
     (Pid :: pid(), message_queue_len) -> {message_queue_len, non_neg_integer()};
     (Pid :: pid(), memory) -> {memory, non_neg_integer()};
@@ -816,9 +818,15 @@ spawn_link(Function) ->
 spawn_link(Module, Function, Args) ->
     erlang:spawn_opt(Module, Function, Args, [link]).
 
+-type heap_growth_strategy() ::
+    bounded_free
+    | minimum
+    | fibonacci.
+
 -type spawn_option() ::
     {min_heap_size, pos_integer()}
     | {max_heap_size, pos_integer()}
+    | {heap_growth, heap_growth_strategy()}
     | link
     | monitor.
 
diff --git a/src/libAtomVM/context.c b/src/libAtomVM/context.c
index 5e21308cc..423c2cb2a 100644
--- a/src/libAtomVM/context.c
+++ b/src/libAtomVM/context.c
@@ -66,6 +66,7 @@ Context *context_new(GlobalContext *glb)
 
     ctx->min_heap_size = 0;
     ctx->max_heap_size = 0;
+    ctx->heap_growth_strategy = BoundedFreeHeapGrowth;
     ctx->has_min_heap_size = 0;
     ctx->has_max_heap_size = 0;
 
@@ -236,6 +237,7 @@ bool context_get_process_info(Context *ctx, term *out, term atom_key)
     size_t ret_size;
     switch (atom_key) {
         case HEAP_SIZE_ATOM:
+        case TOTAL_HEAP_SIZE_ATOM:
         case STACK_SIZE_ATOM:
         case MESSAGE_QUEUE_LEN_ATOM:
         case MEMORY_ATOM:
@@ -268,7 +270,15 @@ bool context_get_process_info(Context *ctx, term *out, term atom_key)
         // heap_size size in words of the heap of the process
         case HEAP_SIZE_ATOM: {
             term_put_tuple_element(ret, 0, HEAP_SIZE_ATOM);
-            unsigned long value = memory_heap_memory_size(&ctx->heap) - context_stack_size(ctx);
+            unsigned long value = memory_heap_youngest_size(&ctx->heap);
+            term_put_tuple_element(ret, 1, term_from_int32(value));
+            break;
+        }
+
+        // total_heap_size size in words of the heap of the process, including fragments
+        case TOTAL_HEAP_SIZE_ATOM: {
+            term_put_tuple_element(ret, 0, TOTAL_HEAP_SIZE_ATOM);
+            unsigned long value = memory_heap_memory_size(&ctx->heap);
             term_put_tuple_element(ret, 1, term_from_int32(value));
             break;
         }
diff --git a/src/libAtomVM/context.h b/src/libAtomVM/context.h
index 71286b2ed..c8ef56da2 100644
--- a/src/libAtomVM/context.h
+++ b/src/libAtomVM/context.h
@@ -67,6 +67,13 @@ enum ContextFlags
     Trap = 32,
 };
 
+enum HeapGrowthStrategy
+{
+    BoundedFreeHeapGrowth = 0,
+    MinimumHeapGrowth,
+    FibonacciHeapGrowth
+};
+
 // Max number of x(N) & fr(N) registers
 // BEAM sets this to 1024.
 #define MAX_REG 16
@@ -92,6 +99,7 @@ struct Context
 
     size_t min_heap_size;
     size_t max_heap_size;
+    enum HeapGrowthStrategy heap_growth_strategy;
 
     unsigned long cp;
 
diff --git a/src/libAtomVM/defaultatoms.c b/src/libAtomVM/defaultatoms.c
index dc0035e2c..baa07bb3d 100644
--- a/src/libAtomVM/defaultatoms.c
+++ b/src/libAtomVM/defaultatoms.c
@@ -149,6 +149,12 @@ static const char *const kill_atom = "\x4" "kill";
 static const char *const killed_atom = "\x6" "killed";
 static const char *const links_atom = "\x5" "links";
 
+static const char *const total_heap_size_atom = "\xF" "total_heap_size";
+static const char *const heap_growth_atom = "\xB" "heap_growth";
+static const char *const bounded_free_atom = "\xC" "bounded_free";
+static const char *const minimum_atom = "\x7" "minimum";
+static const char *const fibonacci_atom = "\x9" "fibonacci";
+
 void defaultatoms_init(GlobalContext *glb)
 {
     int ok = 1;
@@ -282,6 +288,12 @@ void defaultatoms_init(GlobalContext *glb)
     ok &= globalcontext_insert_atom(glb, killed_atom) == KILLED_ATOM_INDEX;
     ok &= globalcontext_insert_atom(glb, links_atom) == LINKS_ATOM_INDEX;
 
+    ok &= globalcontext_insert_atom(glb, total_heap_size_atom) == TOTAL_HEAP_SIZE_ATOM_INDEX;
+    ok &= globalcontext_insert_atom(glb, heap_growth_atom) == HEAP_GROWTH_ATOM_INDEX;
+    ok &= globalcontext_insert_atom(glb, bounded_free_atom) == BOUNDED_FREE_ATOM_INDEX;
+    ok &= globalcontext_insert_atom(glb, minimum_atom) == MINIMUM_ATOM_INDEX;
+    ok &= globalcontext_insert_atom(glb, fibonacci_atom) == FIBONACCI_ATOM_INDEX;
+
     if (!ok) {
         AVM_ABORT();
     }
diff --git a/src/libAtomVM/defaultatoms.h b/src/libAtomVM/defaultatoms.h
index 220d72c15..f4a0623af 100644
--- a/src/libAtomVM/defaultatoms.h
+++ b/src/libAtomVM/defaultatoms.h
@@ -158,7 +158,13 @@ extern "C" {
 #define KILLED_ATOM_INDEX 101
 #define LINKS_ATOM_INDEX 102
 
-#define PLATFORM_ATOMS_BASE_INDEX 103
+#define TOTAL_HEAP_SIZE_ATOM_INDEX 103
+#define HEAP_GROWTH_ATOM_INDEX 104
+#define BOUNDED_FREE_ATOM_INDEX 105
+#define MINIMUM_ATOM_INDEX 106
+#define FIBONACCI_ATOM_INDEX 107
+
+#define PLATFORM_ATOMS_BASE_INDEX 108
 
 #define FALSE_ATOM TERM_FROM_ATOM_INDEX(FALSE_ATOM_INDEX)
 #define TRUE_ATOM TERM_FROM_ATOM_INDEX(TRUE_ATOM_INDEX)
@@ -291,6 +297,12 @@ extern "C" {
 #define KILLED_ATOM TERM_FROM_ATOM_INDEX(KILLED_ATOM_INDEX)
 #define LINKS_ATOM TERM_FROM_ATOM_INDEX(LINKS_ATOM_INDEX)
 
+#define TOTAL_HEAP_SIZE_ATOM TERM_FROM_ATOM_INDEX(TOTAL_HEAP_SIZE_ATOM_INDEX)
+#define HEAP_GROWTH_ATOM TERM_FROM_ATOM_INDEX(HEAP_GROWTH_ATOM_INDEX)
+#define BOUNDED_FREE_ATOM TERM_FROM_ATOM_INDEX(BOUNDED_FREE_ATOM_INDEX)
+#define MINIMUM_ATOM TERM_FROM_ATOM_INDEX(MINIMUM_ATOM_INDEX)
+#define FIBONACCI_ATOM TERM_FROM_ATOM_INDEX(FIBONACCI_ATOM_INDEX)
+
 void defaultatoms_init(GlobalContext *glb);
 
 void platform_defaultatoms_init(GlobalContext *glb);
diff --git a/src/libAtomVM/memory.c b/src/libAtomVM/memory.c
index 2276333bc..2d622b6bb 100644
--- a/src/libAtomVM/memory.c
+++ b/src/libAtomVM/memory.c
@@ -34,6 +34,7 @@
 //#define ENABLE_TRACE
 
 #include "trace.h"
+#include "utils.h"
 
 #ifndef MAX
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
@@ -101,6 +102,22 @@ enum MemoryGCResult memory_erl_nif_env_ensure_free(ErlNifEnv *env, size_t size)
     return MEMORY_GC_OK;
 }
 
+// Follow Erlang/OTP 18 fibonacci series.
+static size_t next_fibonacci_heap_size(size_t size)
+{
+    static const size_t fib_seq[] = { 12, 38, 51, 90, 142, 233, 376, 610, 987, 1598, 2586, 4185, 6772, 10958,
+        17731, 28690, 46422, 75113, 121536, 196650, 318187, 514838, 833026,
+        1347865, 2180892, 3528758, 5709651 };
+    for (size_t i = 0; i < sizeof(fib_seq) / sizeof(fib_seq[0]); i++) {
+        if (size <= fib_seq[i]) {
+            return fib_seq[i];
+        }
+    }
+    return size + size / 5;
+}
+
+#define FIBONACCI_HEAP_GROWTH_REDUCTION_THRESHOLD 10000
+
 enum MemoryGCResult memory_ensure_free_with_roots(Context *c, size_t size, size_t num_roots, term *roots, enum MemoryAllocMode alloc_mode)
 {
     size_t free_space = context_avail_free_memory(c);
@@ -109,22 +126,87 @@ enum MemoryGCResult memory_ensure_free_with_roots(Context *c, size_t size, size_
             return memory_heap_alloc_new_fragment(&c->heap, size);
         }
     } else {
-        size_t maximum_free_space = 2 * (size + MIN_FREE_SPACE_SIZE);
-        if (free_space < size || (alloc_mode == MEMORY_FORCE_SHRINK) || ((alloc_mode == MEMORY_CAN_SHRINK) && free_space > maximum_free_space)) {
-            size_t memory_size = memory_heap_memory_size(&c->heap);
-            if (UNLIKELY(memory_gc(c, memory_size + size + MIN_FREE_SPACE_SIZE, num_roots, roots) != MEMORY_GC_OK)) {
-                // TODO: handle this more gracefully
-                TRACE("Unable to allocate memory for GC.  memory_size=%zu size=%u\n", memory_size, size);
-                return MEMORY_GC_ERROR_FAILED_ALLOCATION;
+        // Target heap size depends on:
+        // - alloc_mode (MEMORY_FORCE_SHRINK takes precedence)
+        // - heap growth strategy
+        bool should_gc = free_space < size || (alloc_mode == MEMORY_FORCE_SHRINK);
+        size_t memory_size = 0;
+        if (!should_gc) {
+            switch (c->heap_growth_strategy) {
+                case BoundedFreeHeapGrowth: {
+                    size_t maximum_free_space = 2 * (size + MIN_FREE_SPACE_SIZE);
+                    should_gc = ((alloc_mode == MEMORY_CAN_SHRINK) && free_space - size > maximum_free_space);
+                } break;
+                case MinimumHeapGrowth:
+                    should_gc = ((alloc_mode == MEMORY_CAN_SHRINK) && free_space - size > 0);
+                    break;
+                case FibonacciHeapGrowth: {
+                    memory_size = memory_heap_memory_size(&c->heap);
+                    should_gc = ((alloc_mode == MEMORY_CAN_SHRINK) && free_space - size > 3 * memory_size / 4);
+                    break;
+                }
+            }
+        }
+        if (should_gc) {
+            if (memory_size == 0) {
+                memory_size = memory_heap_memory_size(&c->heap);
+            }
+            size_t target_size;
+            switch (c->heap_growth_strategy) {
+                case BoundedFreeHeapGrowth:
+                    if (free_space < size) {
+                        target_size = memory_size + size + MIN_FREE_SPACE_SIZE;
+                    } else {
+                        size_t maximum_free_space = 2 * (size + MIN_FREE_SPACE_SIZE);
+                        target_size = memory_size - free_space + maximum_free_space;
+                    }
+                    break;
+                case MinimumHeapGrowth:
+                    target_size = memory_size - free_space + size;
+                    break;
+                case FibonacciHeapGrowth:
+                    target_size = next_fibonacci_heap_size(memory_size - free_space + size);
+                    break;
+                default:
+                    UNREACHABLE();
             }
-            if (alloc_mode != MEMORY_NO_SHRINK) {
+            target_size = MAX(c->has_min_heap_size ? c->min_heap_size : 0, target_size);
+            if (target_size != memory_size) {
+                if (UNLIKELY(memory_gc(c, target_size, num_roots, roots) != MEMORY_GC_OK)) {
+                    // TODO: handle this more gracefully
+                    TRACE("Unable to allocate memory for GC.  target_size=%zu\n", target_size);
+                    return MEMORY_GC_ERROR_FAILED_ALLOCATION;
+                }
+                should_gc = alloc_mode == MEMORY_FORCE_SHRINK;
+                size_t new_memory_size = memory_heap_memory_size(&c->heap);
+                size_t new_target_size = new_memory_size;
                 size_t new_free_space = context_avail_free_memory(c);
-                if (new_free_space > maximum_free_space) {
-                    size_t new_memory_size = memory_heap_memory_size(&c->heap);
-                    size_t new_requested_size = (new_memory_size - new_free_space) + maximum_free_space;
-                    if (!c->has_min_heap_size || (c->min_heap_size < new_requested_size)) {
-                        if (UNLIKELY(memory_gc(c, new_requested_size, num_roots, roots) != MEMORY_GC_OK)) {
-                            TRACE("Unable to allocate memory for GC shrink.  new_memory_size=%zu new_free_space=%zu new_minimum_free_space=%zu size=%u\n", new_memory_size, new_free_space, maximum_free_space, size);
+                switch (c->heap_growth_strategy) {
+                    case BoundedFreeHeapGrowth: {
+                        size_t maximum_free_space = 2 * (size + MIN_FREE_SPACE_SIZE);
+                        should_gc = should_gc || (alloc_mode != MEMORY_NO_SHRINK && new_free_space > maximum_free_space);
+                        if (should_gc) {
+                            new_target_size = (new_memory_size - new_free_space) + maximum_free_space;
+                        }
+                    } break;
+                    case MinimumHeapGrowth:
+                        should_gc = should_gc || (alloc_mode != MEMORY_NO_SHRINK && new_free_space > 0);
+                        if (should_gc) {
+                            new_target_size = new_memory_size - new_free_space + size;
+                        }
+                        break;
+                    case FibonacciHeapGrowth:
+                        should_gc = should_gc || (new_memory_size > FIBONACCI_HEAP_GROWTH_REDUCTION_THRESHOLD && new_free_space >= 3 * new_memory_size / 4);
+                        if (should_gc) {
+                            target_size = next_fibonacci_heap_size(new_memory_size - new_free_space + size);
+                        }
+                        break;
+                }
+                if (should_gc) {
+                    new_target_size = MAX(c->has_min_heap_size ? c->min_heap_size : 0, new_target_size);
+                    if (new_target_size != new_memory_size) {
+                        if (UNLIKELY(memory_gc(c, new_target_size, num_roots, roots) != MEMORY_GC_OK)) {
+                            TRACE("Unable to allocate memory for GC shrink.  new_memory_size=%zu new_free_space=%zu size=%u\n", new_memory_size, new_free_space, size);
                             return MEMORY_GC_ERROR_FAILED_ALLOCATION;
                         }
                     }
diff --git a/src/libAtomVM/memory.h b/src/libAtomVM/memory.h
index f1fd9fa5f..614a351f1 100644
--- a/src/libAtomVM/memory.h
+++ b/src/libAtomVM/memory.h
@@ -151,6 +151,18 @@ static inline size_t memory_heap_fragment_memory_size(const HeapFragment *fragme
     return result;
 }
 
+/**
+ * @brief return the size of the youngest generation of the heap.
+ * @details in some condition, this function returns the size of a fragment
+ * where the stack is not.
+ * @param heap the heap to get the youngest size of
+ * @returns the size in terms
+ */
+static inline size_t memory_heap_youngest_size(const Heap *heap)
+{
+    return heap->heap_end - heap->heap_start;
+}
+
 /**
  * @brief return the total memory size of a heap, including fragments.
  *
@@ -159,7 +171,7 @@ static inline size_t memory_heap_fragment_memory_size(const HeapFragment *fragme
  */
 static inline size_t memory_heap_memory_size(const Heap *heap)
 {
-    size_t result = heap->heap_end - heap->heap_start;
+    size_t result = memory_heap_youngest_size(heap);
     if (heap->root->next) {
         result += memory_heap_fragment_memory_size(heap->root->next);
     }
diff --git a/src/libAtomVM/nifs.c b/src/libAtomVM/nifs.c
index 0ced5b890..48aeed4ce 100644
--- a/src/libAtomVM/nifs.c
+++ b/src/libAtomVM/nifs.c
@@ -1078,6 +1078,7 @@ static term do_spawn(Context *ctx, Context *new_ctx, term opts_term)
     term max_heap_size_term = interop_proplist_get_value(opts_term, MAX_HEAP_SIZE_ATOM);
     term link_term = interop_proplist_get_value(opts_term, LINK_ATOM);
     term monitor_term = interop_proplist_get_value(opts_term, MONITOR_ATOM);
+    term heap_growth_strategy = interop_proplist_get_value_default(opts_term, HEAP_GROWTH_ATOM, BOUNDED_FREE_ATOM);
 
     if (min_heap_size_term != term_nil()) {
         if (UNLIKELY(!term_is_integer(min_heap_size_term))) {
@@ -1087,8 +1088,6 @@ static term do_spawn(Context *ctx, Context *new_ctx, term opts_term)
         }
         new_ctx->has_min_heap_size = 1;
         new_ctx->min_heap_size = term_to_int(min_heap_size_term);
-    } else {
-        min_heap_size_term = term_from_int(0);
     }
     if (max_heap_size_term != term_nil()) {
         if (UNLIKELY(!term_is_integer(max_heap_size_term))) {
@@ -1100,12 +1099,26 @@ static term do_spawn(Context *ctx, Context *new_ctx, term opts_term)
     }
 
     if (new_ctx->has_min_heap_size && new_ctx->has_max_heap_size) {
-        if (term_to_int(min_heap_size_term) > term_to_int(max_heap_size_term)) {
+        if (new_ctx->min_heap_size > new_ctx->max_heap_size) {
             context_destroy(new_ctx);
             RAISE_ERROR(BADARG_ATOM);
         }
     }
 
+    switch (heap_growth_strategy) {
+        case BOUNDED_FREE_ATOM:
+            new_ctx->heap_growth_strategy = BoundedFreeHeapGrowth;
+            break;
+        case MINIMUM_ATOM:
+            new_ctx->heap_growth_strategy = MinimumHeapGrowth;
+            break;
+        case FIBONACCI_ATOM:
+            new_ctx->heap_growth_strategy = FibonacciHeapGrowth;
+            break;
+        default:
+            context_destroy(new_ctx);
+            RAISE_ERROR(BADARG_ATOM);
+    }
     uint64_t ref_ticks = 0;
 
     if (link_term == TRUE_ATOM) {
@@ -1254,8 +1267,11 @@ static term nif_erlang_spawn_opt(Context *ctx, int argc, term argv[])
             RAISE_ERROR(BADARG_ATOM);
         }
         min_heap_size = term_to_int(min_heap_size_term);
+        new_ctx->has_min_heap_size = true;
+        new_ctx->min_heap_size = min_heap_size;
     }
-    avm_int_t size = MAX(min_heap_size, memory_estimate_usage(args_term));
+
+    avm_int_t size = memory_estimate_usage(args_term);
     if (UNLIKELY(memory_ensure_free_opt(new_ctx, size, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) {
         // Context was not scheduled yet, we can destroy it.
         context_destroy(new_ctx);
diff --git a/tests/erlang_tests/CMakeLists.txt b/tests/erlang_tests/CMakeLists.txt
index 2204f797c..e660dec10 100644
--- a/tests/erlang_tests/CMakeLists.txt
+++ b/tests/erlang_tests/CMakeLists.txt
@@ -169,6 +169,7 @@ compile_erlang(test_func_info2)
 compile_erlang(test_func_info3)
 compile_erlang(test_process_info)
 compile_erlang(test_min_heap_size)
+compile_erlang(test_heap_growth)
 compile_erlang(test_system_flag)
 compile_erlang(test_system_info)
 compile_erlang(test_binary_to_term)
@@ -610,6 +611,7 @@ add_custom_target(erlang_test_modules DEPENDS
     test_func_info3.beam
     test_process_info.beam
     test_min_heap_size.beam
+    test_heap_growth.beam
     test_system_flag.beam
     test_system_info.beam
     test_binary_to_term.beam
diff --git a/tests/erlang_tests/test_heap_growth.erl b/tests/erlang_tests/test_heap_growth.erl
new file mode 100644
index 000000000..4ad544bf4
--- /dev/null
+++ b/tests/erlang_tests/test_heap_growth.erl
@@ -0,0 +1,216 @@
+%
+% This file is part of AtomVM.
+%
+% Copyright 2023 Paul Guyot <pguyot@kallisys.net>
+%
+% Licensed under the Apache License, Version 2.0 (the "License");
+% you may not use this file except in compliance with the License.
+% You may obtain a copy of the License at
+%
+%    http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS,
+% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+% See the License for the specific language governing permissions and
+% limitations under the License.
+%
+% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
+%
+
+-module(test_heap_growth).
+
+-export([start/0]).
+
+start() ->
+    ok = test_grow_beyond_min_heap_size(),
+    ok = test_bounded_free_strategy(false),
+    ok = test_bounded_free_strategy(true),
+    ok = test_minimum_strategy(),
+    ok = test_fibonacci_strategy(),
+    0.
+
+test_grow_beyond_min_heap_size() ->
+    {Pid1, Ref1} = spawn_opt(
+        fun() ->
+            % Heap size is set to minimum at first GC/Heap growth
+            alloc_some_heap_words(10),
+            {total_heap_size, 100} = process_info(self(), total_heap_size),
+            Var = alloc_some_heap_words(200),
+            {total_heap_size, X} = process_info(self(), total_heap_size),
+            true = X > 100,
+            % do something with Var to avoid compiler optimizations
+            true = 200 =:= length(Var)
+        end,
+        [monitor, {min_heap_size, 100}]
+    ),
+    ok =
+        receive
+            {'DOWN', Ref1, process, Pid1, normal} -> ok
+        after 500 -> timeout
+        end,
+    ok.
+
+test_bounded_free_strategy(UseDefault) ->
+    Opt =
+        if
+            UseDefault -> [];
+            true -> [{heap_growth, bounded_free}]
+        end,
+    {Pid1, Ref1} = spawn_opt(
+        fun() ->
+            {total_heap_size, X1} = process_info(self(), total_heap_size),
+            ok = test_growth_bounded(32),
+            {total_heap_size, X2} = process_info(self(), total_heap_size),
+            % Allocate again, this is when heap will be shrunk
+            Var1 = alloc_some_heap_words(10),
+            {total_heap_size, X3} = process_info(self(), total_heap_size),
+            20 = erts_debug:flat_size(Var1),
+            true = X3 < X2,
+            true = X3 - X1 - erts_debug:flat_size(Var1) < 32
+        end,
+        [monitor | Opt]
+    ),
+    ok =
+        receive
+            {'DOWN', Ref1, process, Pid1, normal} -> ok
+        after 500 -> timeout
+        end,
+    ok.
+
+test_minimum_strategy() ->
+    {Pid1, Ref1} = spawn_opt(
+        fun() ->
+            {total_heap_size, X1} = process_info(self(), total_heap_size),
+            % Cannot really be 0 because we allocate more than just the list.
+            ok = test_growth_bounded(10),
+            {total_heap_size, X2} = process_info(self(), total_heap_size),
+            % Allocate again, this is when heap will be shrunk
+            Var1 = alloc_some_heap_words(10),
+            {total_heap_size, X3} = process_info(self(), total_heap_size),
+            20 = erts_debug:flat_size(Var1),
+            true = X3 < X2,
+            true = X3 - X1 - erts_debug:flat_size(Var1) =< 10
+        end,
+        [monitor, {heap_growth, minimum}]
+    ),
+    ok =
+        receive
+            {'DOWN', Ref1, process, Pid1, normal} -> ok
+        after 500 -> timeout
+        end,
+    ok.
+
+% This test is a little bit long on the CI
+% It aims to test:
+% - that heap sizes progress following fibonacci suite like progression
+% - until a given size where they progress slower
+% - and in both cases, memory is eventually reclaimed
+test_fibonacci_strategy() ->
+    % Test small increments follow fibonacci
+    {Pid1, Ref1} = spawn_opt(
+        fun() ->
+            MaxHeap = test_growth_fibonacci(50, 20000),
+            % Allocate again until heap is shrunk
+            NewHeap = allocate_until_heap_size_changes(MaxHeap),
+            true = NewHeap < MaxHeap
+        end,
+        [monitor, link, {heap_growth, fibonacci}]
+    ),
+    % Test large increments no longer follow fibonacci
+    {Pid2, Ref2} = spawn_opt(
+        fun() ->
+            MaxHeap = test_growth_fibonacci(50000, 10000000),
+            % Allocate again until heap is shrunk
+            NewHeap = allocate_until_heap_size_changes(MaxHeap),
+            true = NewHeap < MaxHeap
+        end,
+        [monitor, link, {heap_growth, fibonacci}]
+    ),
+    ok =
+        receive
+            {'DOWN', Ref1, process, Pid1, normal} -> ok
+        after 300000 -> timeout
+        end,
+    ok =
+        receive
+            {'DOWN', Ref2, process, Pid2, normal} -> ok
+        after 300000 -> timeout
+        end,
+    ok.
+
+alloc_some_heap_words(N) ->
+    alloc_some_heap_words(N, []).
+
+alloc_some_heap_words(0, Acc) -> Acc;
+alloc_some_heap_words(N, Acc) -> alloc_some_heap_words(N - 1, [N | Acc]).
+
+test_growth_bounded(Boundary) ->
+    % Test growth is bounded
+    Var1 = alloc_some_heap_words(10),
+    {total_heap_size, X1} = process_info(self(), total_heap_size),
+    Var2 = alloc_some_heap_words(10),
+    {total_heap_size, X2} = process_info(self(), total_heap_size),
+    Var3 = alloc_some_heap_words(10),
+    {total_heap_size, X3} = process_info(self(), total_heap_size),
+    true = X2 - X1 - erts_debug:flat_size(Var2) =< Boundary,
+    true = X3 - X2 - erts_debug:flat_size(Var3) =< Boundary,
+    true = X3 - X1 - erts_debug:flat_size(Var2) - erts_debug:flat_size(Var3) =< Boundary,
+    20 = erts_debug:flat_size(Var1),
+    ok.
+
+test_growth_fibonacci(Increment, TargetMax) ->
+    HeapSizes = collect_heap_sizes([], [], Increment, TargetMax),
+    [MaxHeap | _] = HeapSizes,
+    ok = test_fibonacci_heap_sizes(Increment, HeapSizes),
+    MaxHeap.
+
+collect_heap_sizes(Acc, Terms, Increment, TargetMax) ->
+    {total_heap_size, S1} = process_info(self(), total_heap_size),
+    % Heap should not shrink
+    NewAcc =
+        case Acc of
+            [] -> [S1 | Acc];
+            [S1 | _Tail] -> Acc;
+            [SmallerSize | _Tail] when SmallerSize < S1 -> [S1 | Acc]
+        end,
+    if
+        S1 >= TargetMax ->
+            NewAcc;
+        true ->
+            collect_heap_sizes(
+                NewAcc, [alloc_some_heap_words(Increment) | Terms], Increment, TargetMax
+            )
+    end.
+
+test_fibonacci_heap_sizes(Increment, [HeapSize, Previous | Tail]) when HeapSize > 5709651 ->
+    Diff = HeapSize - Previous,
+    Delta = Diff - Previous div 5,
+    true = Delta >= -4 andalso Delta =< 4,
+    test_fibonacci_heap_sizes(Increment, [Previous | Tail]);
+test_fibonacci_heap_sizes(Increment, [_HeapSize, _P1, P2 | _]) when 4 * Increment > P2 ->
+    % * 4 because alloc_some_heap_words
+    % - alloc_some_heap_words allocates approximatively twice the increment
+    % - a number can be skipped because of this
+    % Typically, with an increment of 2000, heaps are 17731,10958,6772,4185,2586
+    % instead of: 17731,10958,4185,2586
+    ok;
+test_fibonacci_heap_sizes(Increment, [HeapSize, P1, P2 | Tail]) when P2 > 233 ->
+    Diff = HeapSize - P1,
+    Delta = Diff - P2,
+    true = Delta >= -4 andalso Delta =< 4,
+    test_fibonacci_heap_sizes(Increment, [P1, P2 | Tail]);
+test_fibonacci_heap_sizes(_Increment, [610, 376, 233 | _]) ->
+    ok;
+test_fibonacci_heap_sizes(_Increment, HeapSizes) ->
+    {unexpected, HeapSizes}.
+
+allocate_until_heap_size_changes(Heap) ->
+    {total_heap_size, S1} = process_info(self(), total_heap_size),
+    if
+        S1 =/= Heap ->
+            S1;
+        true ->
+            alloc_some_heap_words(100),
+            allocate_until_heap_size_changes(Heap)
+    end.
diff --git a/tests/erlang_tests/test_min_heap_size.erl b/tests/erlang_tests/test_min_heap_size.erl
index de7ba3816..e27cf7f8e 100644
--- a/tests/erlang_tests/test_min_heap_size.erl
+++ b/tests/erlang_tests/test_min_heap_size.erl
@@ -20,53 +20,37 @@
 
 -module(test_min_heap_size).
 
--export([start/0, loop/1]).
+-export([start/0]).
 
 start() ->
-    Self = self(),
-    Pid1 = spawn_opt(?MODULE, loop, [Self], []),
-    receive
-        ok -> ok
-    end,
-    receive
-    after 100 ->
-        ok
-    end,
-    {memory, Pid1MemorySize} = process_info(Pid1, memory),
-    case erlang:system_info(machine) of
-        "BEAM" -> ok;
-        _ -> assert(Pid1MemorySize < 1024)
-    end,
-    Pid2 = spawn_opt(?MODULE, loop, [Self], [{min_heap_size, 1024}]),
-    receive
-        ok -> ok
-    end,
-    receive
-    after 100 ->
-        ok
-    end,
-    {memory, Pid2MemorySize} = process_info(Pid2, memory),
-    assert(1024 =< Pid2MemorySize),
-    Pid1 ! {Self, stop},
-    receive
-        ok -> ok
-    end,
-    Pid2 ! {Self, stop},
-    receive
-        ok -> ok
-    end,
+    ok = test_min_heap_size(1000),
+    ok = test_min_heap_size(5000),
     0.
 
-loop(undefined) ->
-    receive
-        {Pid, stop} ->
-            Pid ! ok
-    after 10 ->
-        erlang:garbage_collect(),
-        loop(undefined)
-    end;
-loop(Pid) ->
-    Pid ! ok,
-    loop(undefined).
+test_min_heap_size(MinSize) ->
+    {Pid1, Ref1} = spawn_opt(
+        fun() ->
+            % Heap size is set to minimum at first GC/Heap growth
+            alloc_some_heap_words(),
+            {total_heap_size, TotalHeapSize} = process_info(self(), total_heap_size),
+            case erlang:system_info(machine) of
+                "BEAM" ->
+                    true = TotalHeapSize >= MinSize;
+                _ ->
+                    TotalHeapSize = MinSize
+            end
+        end,
+        [monitor, {min_heap_size, MinSize}]
+    ),
+    ok =
+        receive
+            {'DOWN', Ref1, process, Pid1, normal} -> ok
+        after 500 -> timeout
+        end,
+    ok.
 
-assert(true) -> ok.
+alloc_some_heap_words() ->
+    alloc_some_heap_words(20, []).
+
+alloc_some_heap_words(0, _Acc) -> ok;
+alloc_some_heap_words(N, Acc) -> alloc_some_heap_words(N - 1, [N | Acc]).
diff --git a/tests/erlang_tests/test_process_info.erl b/tests/erlang_tests/test_process_info.erl
index 987898619..6f9f7c5d9 100644
--- a/tests/erlang_tests/test_process_info.erl
+++ b/tests/erlang_tests/test_process_info.erl
@@ -51,6 +51,7 @@ test_message_queue_len(Pid, Self) ->
     {message_queue_len, MessageQueueLen} = process_info(Pid, message_queue_len),
     {memory, Memory} = process_info(Pid, memory),
     {heap_size, HeapSize} = process_info(Pid, heap_size),
+    {total_heap_size, TotalHeapSize} = process_info(Pid, total_heap_size),
     Pid ! incr,
     Pid ! incr,
     Pid ! incr,
@@ -61,15 +62,20 @@ test_message_queue_len(Pid, Self) ->
     receive
         pong -> ok
     end,
+    {total_heap_size, TotalHeapSize2} = process_info(Pid, total_heap_size),
     {heap_size, HeapSize2} = process_info(Pid, heap_size),
     assert(MessageQueueLen < MessageQueueLen2),
     case erlang:system_info(machine) of
         "BEAM" ->
             assert(Memory =< Memory2),
-            assert(HeapSize =< HeapSize2);
+            assert(HeapSize =< TotalHeapSize),
+            assert(HeapSize2 =< TotalHeapSize2),
+            assert(TotalHeapSize =< TotalHeapSize2);
         _ ->
             assert(Memory < Memory2),
-            assert(HeapSize < HeapSize2)
+            assert(HeapSize =< TotalHeapSize),
+            assert(HeapSize2 =< TotalHeapSize2),
+            assert(TotalHeapSize < TotalHeapSize2)
     end.
 
 loop(undefined, Accum) ->
diff --git a/tests/erlang_tests/test_refc_binaries.erl b/tests/erlang_tests/test_refc_binaries.erl
index ec26bc2ac..a33d74cca 100644
--- a/tests/erlang_tests/test_refc_binaries.erl
+++ b/tests/erlang_tests/test_refc_binaries.erl
@@ -29,6 +29,8 @@
     "01234567890123456789012345678901234567890123456789"
     "01234567890123456789012345678901234567890123456789"
     "01234567890123456789012345678901234567890123456789"
+    "01234567890123456789012345678901234567890123456789"
+    "01234567890123456789012345678901234567890123456789"
 >>).
 
 -define(VERIFY(X),
@@ -179,7 +181,7 @@ get_largest_heap_binary_size() ->
 
 get_heap_size() ->
     erlang:garbage_collect(),
-    {heap_size, Size} = erlang:process_info(self(), heap_size),
+    {total_heap_size, Size} = erlang:process_info(self(), total_heap_size),
     Size * erlang:system_info(wordsize).
 
 get_heap_size(Pid) ->
diff --git a/tests/erlang_tests/test_sub_binaries.erl b/tests/erlang_tests/test_sub_binaries.erl
index f5d7a80b3..de5052b16 100644
--- a/tests/erlang_tests/test_sub_binaries.erl
+++ b/tests/erlang_tests/test_sub_binaries.erl
@@ -29,6 +29,8 @@
     "01234567890123456789012345678901234567890123456789"
     "01234567890123456789012345678901234567890123456789"
     "01234567890123456789012345678901234567890123456789"
+    "01234567890123456789012345678901234567890123456789"
+    "01234567890123456789012345678901234567890123456789"
 >>).
 
 -record(state, {
@@ -291,7 +293,7 @@ get_largest_heap_binary_size() ->
 
 get_heap_size() ->
     erlang:garbage_collect(),
-    {heap_size, Size} = erlang:process_info(self(), heap_size),
+    {total_heap_size, Size} = erlang:process_info(self(), total_heap_size),
     Size * erlang:system_info(wordsize).
 
 get_heap_size(Pid) ->
diff --git a/tests/test.c b/tests/test.c
index 4b7111b04..c5b4800ca 100644
--- a/tests/test.c
+++ b/tests/test.c
@@ -185,6 +185,7 @@ struct Test tests[] = {
     TEST_CASE_EXPECTED(test_func_info3, 120),
     TEST_CASE(test_process_info),
     TEST_CASE(test_min_heap_size),
+    TEST_CASE_ATOMVM_ONLY(test_heap_growth, 0),
     TEST_CASE(test_system_flag),
     TEST_CASE(test_system_info),
     TEST_CASE_EXPECTED(test_funs0, 20),