Three hot-path fixes for atom-heavier code

CHANGELOG.md

@@ -26,6 +26,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added USB CDC port drivers for ESP32, RP2, and STM32 platforms
 
 ### Changed
+- Exact-equality opcodes (`is_eq_exact`, `is_not_eq_exact`, `select_val`) decide two immediates
+  without `term_compare`; atom `case` clauses no longer compare atom names through the atom table
+- Heap fragments (decoded literals, NIF results, messages) no longer force a collection at the next
+  return / NIF call / allocation; they are folded in once large or at the next natural collection
+- `maps:find/2` and `maps:get/3` no longer raise and catch an exception on a missing key
 - Updated network type db() to dbm() to reflect the actual representation of the type
 - Use ES6 modules for emscripten port, using .mjs suffix
 - `ahttp_client` now returns `{error, {parser, incomplete_response}}` when a socket closes mid-response

libs/estdlib/src/maps.erl

@@ -104,13 +104,13 @@ get(Key, Map) ->
 %% @end
 %%-----------------------------------------------------------------------------
 -spec get(Key, Map :: #{Key => Value}, Default :: Value) -> Value.
-get(Key, Map, Default) ->
-    try
-        ?MODULE:get(Key, Map)
-    catch
-        error:{badkey, _} ->
-            Default
-    end.
+get(Key, Map, Default) when is_map(Map) ->
+    case erlang:is_map_key(Key, Map) of
+        true -> erlang:map_get(Key, Map);
+        false -> Default
+    end;
+get(_Key, Map, _Default) ->
+    error({badmap, Map}).
 
 %%-----------------------------------------------------------------------------
 %% @param   Key     the key
@@ -291,13 +291,13 @@ size(Map) ->
 %% @end
 %%-----------------------------------------------------------------------------
 -spec find(Key, Map :: #{Key => Value}) -> {ok, Value} | error.
-find(Key, Map) ->
-    try
-        {ok, ?MODULE:get(Key, Map)}
-    catch
-        _:{badkey, _} ->
-            error
-    end.
+find(Key, Map) when is_map(Map) ->
+    case erlang:is_map_key(Key, Map) of
+        true -> {ok, erlang:map_get(Key, Map)};
+        false -> error
+    end;
+find(_Key, Map) ->
+    error({badmap, Map}).
 
 %%-----------------------------------------------------------------------------
 %% @param   Pred    a function used to filter entries from the map

src/libAtomVM/erl_nif_priv.h

@@ -63,6 +63,7 @@ static inline void erl_nif_env_partial_init_from_globalcontext(ErlNifEnv *env, G
     env->heap.heap_start = NULL;
     env->heap.heap_ptr = NULL;
     env->heap.heap_end = NULL;
+    env->heap.fragments_words = 0;
     env->stack_pointer = NULL;
     env->x[0] = term_nil();
     env->x[1] = term_nil();
@@ -76,6 +77,7 @@ static inline void erl_nif_env_partial_init_from_resource(ErlNifEnv *env, void *
     env->heap.heap_start = NULL;
     env->heap.heap_ptr = NULL;
     env->heap.heap_end = NULL;
+    env->heap.fragments_words = 0;
     env->stack_pointer = NULL;
     env->x[0] = term_nil();
     env->x[1] = term_nil();

src/libAtomVM/jit.c

@@ -514,7 +514,7 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int
                 ctx->e += (n_words + 1);
             }
 
-            if (ctx->heap.root->next) {
+            if (memory_heap_fragments_need_gc(&ctx->heap)) {
                 if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                     return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM);
                 }
@@ -679,7 +679,7 @@ static bool jit_deallocate(Context *ctx, JITState *jit_state, uint32_t n_words)
     ctx->cp = ctx->e[n_words];
     ctx->e += n_words + 1;
     // Hopefully, we only need x[0]
-    if (ctx->heap.root->next) {
+    if (memory_heap_fragments_need_gc(&ctx->heap)) {
         if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
             set_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM);
             return false;
@@ -1210,7 +1210,7 @@ static Context *jit_call_fun(Context *ctx, JITState *jit_state, int offset, term
         if (maybe_call_native(ctx, module_name, function_name, fun_arity, &return_value)) {
             PROCESS_MAYBE_TRAP_RETURN_VALUE(return_value, offset);
             ctx->x[0] = return_value;
-            if (ctx->heap.root->next) {
+            if (memory_heap_fragments_need_gc(&ctx->heap)) {
                 if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                     return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM);
                 }
@@ -1718,7 +1718,7 @@ static Context *jit_apply(Context *ctx, JITState *jit_state, int offset, term mo
     if (maybe_call_native(ctx, module_name, function_name, arity, &native_return)) {
         PROCESS_MAYBE_TRAP_RETURN_VALUE(native_return, offset);
         ctx->x[0] = native_return;
-        if (ctx->heap.root->next) {
+        if (memory_heap_fragments_need_gc(&ctx->heap)) {
             if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                 return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM);
             }

src/libAtomVM/memory.c

@@ -73,6 +73,7 @@ void memory_init_heap_root_fragment(Heap *heap, HeapFragment *root, size_t size)
     heap->root = root;
     root->next = NULL;
     root->mso_list = term_nil();
+    heap->fragments_words = 0;
     heap->heap_start = root->storage;
     heap->heap_ptr = heap->heap_start;
     heap->heap_end = heap->heap_start + size;
@@ -99,10 +100,14 @@ static inline enum MemoryGCResult memory_heap_alloc_new_fragment(Heap *heap, siz
     HeapFragment *root_fragment = heap->root;
     term *old_end = heap->heap_end;
     term mso_list = root_fragment->mso_list;
+    // The old root (holding everything allocated so far) becomes a non-root
+    // fragment below; memory_init_heap resets the running total, so carry it.
+    size_t old_fragments_words = heap->fragments_words + (size_t) (heap->heap_ptr - heap->heap_start);
     if (UNLIKELY(memory_init_heap(heap, size) != MEMORY_GC_OK)) {
         TRACE("Unable to allocate memory fragment.  size=%u\n", (unsigned int) size);
         return MEMORY_GC_ERROR_FAILED_ALLOCATION;
     }
+    heap->fragments_words = old_fragments_words;
     // Convert root fragment to non-root fragment.
     root_fragment->heap_end = old_end; // used to hold mso_list when it was the root fragment
     heap->root->next = root_fragment;
@@ -159,7 +164,10 @@ enum MemoryGCResult memory_ensure_free_with_roots(Context *c, size_t size, size_
         // Target heap size depends on:
         // - alloc_mode (MEMORY_FORCE_SHRINK takes precedence)
         // - heap growth strategy
-        bool should_gc = free_space < size || (alloc_mode == MEMORY_FORCE_SHRINK) || c->heap.root->next != NULL;
+        // Heap fragments (literals decoded from the module literal table, NIF
+        // results, received messages) are folded in only once they are large
+        // (see memory_heap_fragments_need_gc), otherwise at the next natural GC.
+        bool should_gc = free_space < size || (alloc_mode == MEMORY_FORCE_SHRINK) || memory_heap_fragments_need_gc(&c->heap);
         size_t memory_size = 0;
         if (!should_gc) {
             switch (c->heap_growth_strategy) {
@@ -906,6 +914,7 @@ HOT_FUNC static term memory_shallow_copy_term(HeapFragment *old_fragment, term t
 
 void memory_heap_append_fragment(Heap *heap, HeapFragment *fragment, term mso_list)
 {
+    heap->fragments_words += memory_heap_fragment_memory_size(fragment);
     // The fragment we are appending may have next fragments
     // So we take our current next and we add it to the tail of the passed list
     if (heap->root->next) {

src/libAtomVM/memory.h

@@ -85,6 +85,10 @@ struct Heap
     term *heap_start;
     term *heap_ptr;
     term *heap_end;
+    // Running total of the words held in the fragments chained off root
+    // (root->next...), so heap size and the fold-in decision are O(1) per
+    // allocation instead of a chain walk.
+    size_t fragments_words;
 };
 
 #ifndef TYPEDEF_HEAP
@@ -176,11 +180,34 @@ static inline size_t memory_heap_youngest_size(const Heap *heap)
  */
 static inline size_t memory_heap_memory_size(const Heap *heap)
 {
-    size_t result = memory_heap_youngest_size(heap);
-    if (heap->root->next) {
-        result += memory_heap_fragment_memory_size(heap->root->next);
+    // Called on every allocation under the fibonacci growth policy: keep it
+    // O(1) via the running fragment total.
+    return memory_heap_youngest_size(heap) + heap->fragments_words;
+}
+
+/**
+ * @brief Whether the heap's fragments are worth folding in right now.
+ *
+ * @details Fragments come from decoded literals (every use of a compound
+ * literal), NIF results and received messages. Any fragment at all used to
+ * force a shrinking GC at the next return / NIF call / allocation; for code
+ * that references compound literals in nearly every function (Gleam- and
+ * Elixir-compiled code in particular) that was a full copying collection
+ * every few instructions. Fragments are valid heap memory the collector
+ * copies out of like anything else, so they are folded in only once they are
+ * large relative to the heap or in absolute terms; otherwise they wait for
+ * the next natural collection.
+ * @param heap the heap
+ * @return true if a collection should be forced to merge the fragments
+ */
+static inline bool memory_heap_fragments_need_gc(const Heap *heap)
+{
+    if (heap->root->next == NULL) {
+        return false;
     }
-    return result;
+    size_t frag_size = heap->fragments_words;
+    size_t young_size = memory_heap_youngest_size(heap);
+    return frag_size > young_size / 4 || frag_size > 65536;
 }
 
 /**

src/libAtomVM/opcodesswitch.h

@@ -1106,7 +1106,7 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx)
         if (maybe_call_native(ctx, module_name, function_name, fun_arity, &return_value)) { \
             PROCESS_MAYBE_TRAP_RETURN_VALUE(return_value);              \
             x_regs[0] = return_value;                                   \
-            if (ctx->heap.root->next) {                                 \
+            if (memory_heap_fragments_need_gc(&ctx->heap)) {                                 \
                 if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { \
                     RAISE_ERROR(OUT_OF_MEMORY_ATOM);                    \
                 }                                                       \
@@ -1919,7 +1919,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                         term return_value = nif->nif_ptr(ctx, arity, x_regs);
                         PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_PC_INDEX_ARITY(return_value, orig_pc, mod, index, arity);
                         x_regs[0] = return_value;
-                        if (ctx->heap.root->next) {
+                        if (memory_heap_fragments_need_gc(&ctx->heap)) {
                             if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                                 RAISE_ERROR(OUT_OF_MEMORY_ATOM);
                             }
@@ -2044,7 +2044,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                         ctx->cp = ctx->e[n_words];
                         ctx->e += (n_words + 1);
 
-                        if (ctx->heap.root->next) {
+                        if (memory_heap_fragments_need_gc(&ctx->heap)) {
                             if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                                 RAISE_ERROR(OUT_OF_MEMORY_ATOM);
                             }
@@ -2333,7 +2333,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                 ctx->e += n_words + 1;
                 DEBUG_DUMP_STACK(ctx);
                 // Hopefully, we only need x[0]
-                if (ctx->heap.root->next) {
+                if (memory_heap_fragments_need_gc(&ctx->heap)) {
                     if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                         RAISE_ERROR(OUT_OF_MEMORY_ATOM);
                     }
@@ -2617,11 +2617,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
 
                 TRACE("is_eq_exact/3, label=%" PRIu32 ", arg1=%" TERM_X_FMT ", arg2=%" TERM_X_FMT "\n", label, arg1, arg2);
 
-                TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global);
-                if (result & (TermLessThan | TermGreaterThan)) {
-                    pc = mod->labels[label];
-                } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) {
-                    RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+                bool fast_equal;
+                if (term_exact_eq_fast(arg1, arg2, &fast_equal)) {
+                    if (!fast_equal) {
+                        pc = mod->labels[label];
+                    }
+                } else {
+                    TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global);
+                    if (result & (TermLessThan | TermGreaterThan)) {
+                        pc = mod->labels[label];
+                    } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) {
+                        RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+                    }
                 }
 
                 break;
@@ -2637,11 +2644,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
 
                 TRACE("is_not_eq_exact/3, label=%" PRIu32 ", arg1=%" TERM_X_FMT ", arg2=%" TERM_X_FMT "\n", label, arg1, arg2);
 
-                TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global);
-                if (result == TermEquals) {
-                    pc = mod->labels[label];
-                } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) {
-                    RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+                bool fast_equal;
+                if (term_exact_eq_fast(arg1, arg2, &fast_equal)) {
+                    if (fast_equal) {
+                        pc = mod->labels[label];
+                    }
+                } else {
+                    TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global);
+                    if (result == TermEquals) {
+                        pc = mod->labels[label];
+                    } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) {
+                        RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+                    }
                 }
 
                 break;
@@ -2865,12 +2879,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                     DECODE_LABEL(jmp_label, pc)
 
                     if (!jump_to_address) {
-                        TermCompareResult result = term_compare(
-                            src_value, cmp_value, TermCompareExact, ctx->global);
-                        if (result == TermEquals) {
-                            jump_to_address = mod->labels[jmp_label];
-                        } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) {
-                            RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+                        bool fast_equal;
+                        if (term_exact_eq_fast(src_value, cmp_value, &fast_equal)) {
+                            if (fast_equal) {
+                                jump_to_address = mod->labels[jmp_label];
+                            }
+                        } else {
+                            TermCompareResult result = term_compare(
+                                src_value, cmp_value, TermCompareExact, ctx->global);
+                            if (result == TermEquals) {
+                                jump_to_address = mod->labels[jmp_label];
+                            } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) {
+                                RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+                            }
                         }
                     }
                 }
@@ -3135,7 +3156,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb)
                         PROCESS_MAYBE_TRAP_RETURN_VALUE_LAST(return_value);
                         x_regs[0] = return_value;
 
-                        if (ctx->heap.root->next) {
+                        if (memory_heap_fragments_need_gc(&ctx->heap)) {
                             if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                                 RAISE_ERROR(OUT_OF_MEMORY_ATOM);
                             }

src/libAtomVM/scheduler.c

@@ -296,7 +296,7 @@ Context *scheduler_run(GlobalContext *global)
                     if (result->native_handler(result) == NativeContinue) {
                         // If native handler has memory fragments, garbage collect
                         // them
-                        if (result->heap.root->next) {
+                        if (memory_heap_fragments_need_gc(&result->heap)) {
                             if (UNLIKELY(memory_ensure_free_opt(result, 0, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) {
                                 fprintf(stderr, "Out of memory error in native handler\n");
                                 AVM_ABORT();

src/libAtomVM/term.h

@@ -439,6 +439,34 @@ static inline bool term_is_boxed(term t)
     return ((t & TERM_PRIMARY_MASK) == TERM_PRIMARY_BOXED);
 }
 
+/**
+ * @brief Exact-equality fast path for immediates.
+ *
+ * @details Two terms with the same bits are `=:=` equal. Two immediate terms
+ * (atoms, small integers, nil, local pids, ...) with different bits are never
+ * `=:=` equal, because immediates are canonical (a small integer is never
+ * boxed). Only when at least one side is boxed or a list must the caller fall
+ * back to term_compare, which for two different atoms fetches both names from
+ * the atom table and compares them: previously the dominant cost of every
+ * `case` on atoms (OP_SELECT_VAL) in Gleam-compiled code.
+ * @param a first term
+ * @param b second term
+ * @param equal set to the answer when the function returns true
+ * @return true if the answer is decided (in *equal), false if term_compare is needed
+ */
+static inline bool term_exact_eq_fast(term a, term b, bool *equal)
+{
+    if (a == b) {
+        *equal = true;
+        return true;
+    }
+    if (((a & TERM_PRIMARY_MASK) == TERM_PRIMARY_IMMED) && ((b & TERM_PRIMARY_MASK) == TERM_PRIMARY_IMMED)) {
+        *equal = false;
+        return true;
+    }
+    return false;
+}
+
 /**
  * @brief Returns size of a boxed term from its header
  *