diff --git a/CHANGELOG.md b/CHANGELOG.md index 29b81ae4e4..54d2ad46e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added USB CDC port drivers for ESP32, RP2, and STM32 platforms ### Changed +- Exact-equality opcodes (`is_eq_exact`, `is_not_eq_exact`, `select_val`) decide two immediates + without `term_compare`; atom `case` clauses no longer compare atom names through the atom table +- Heap fragments (decoded literals, NIF results, messages) no longer force a collection at the next + return / NIF call / allocation; they are folded in once large or at the next natural collection +- `maps:find/2` and `maps:get/3` no longer raise and catch an exception on a missing key - Updated network type db() to dbm() to reflect the actual representation of the type - Use ES6 modules for emscripten port, using .mjs suffix - `ahttp_client` now returns `{error, {parser, incomplete_response}}` when a socket closes mid-response diff --git a/libs/estdlib/src/maps.erl b/libs/estdlib/src/maps.erl index 2b50d5d524..bca48f3fe4 100644 --- a/libs/estdlib/src/maps.erl +++ b/libs/estdlib/src/maps.erl @@ -104,13 +104,13 @@ get(Key, Map) -> %% @end %%----------------------------------------------------------------------------- -spec get(Key, Map :: #{Key => Value}, Default :: Value) -> Value. -get(Key, Map, Default) -> - try - ?MODULE:get(Key, Map) - catch - error:{badkey, _} -> - Default - end. +get(Key, Map, Default) when is_map(Map) -> + case erlang:is_map_key(Key, Map) of + true -> erlang:map_get(Key, Map); + false -> Default + end; +get(_Key, Map, _Default) -> + error({badmap, Map}). %%----------------------------------------------------------------------------- %% @param Key the key @@ -291,13 +291,13 @@ size(Map) -> %% @end %%----------------------------------------------------------------------------- -spec find(Key, Map :: #{Key => Value}) -> {ok, Value} | error. -find(Key, Map) -> - try - {ok, ?MODULE:get(Key, Map)} - catch - _:{badkey, _} -> - error - end. +find(Key, Map) when is_map(Map) -> + case erlang:is_map_key(Key, Map) of + true -> {ok, erlang:map_get(Key, Map)}; + false -> error + end; +find(_Key, Map) -> + error({badmap, Map}). %%----------------------------------------------------------------------------- %% @param Pred a function used to filter entries from the map diff --git a/src/libAtomVM/erl_nif_priv.h b/src/libAtomVM/erl_nif_priv.h index 008dca8a8b..d22b2b506e 100644 --- a/src/libAtomVM/erl_nif_priv.h +++ b/src/libAtomVM/erl_nif_priv.h @@ -63,6 +63,7 @@ static inline void erl_nif_env_partial_init_from_globalcontext(ErlNifEnv *env, G env->heap.heap_start = NULL; env->heap.heap_ptr = NULL; env->heap.heap_end = NULL; + env->heap.fragments_words = 0; env->stack_pointer = NULL; env->x[0] = term_nil(); env->x[1] = term_nil(); @@ -76,6 +77,7 @@ static inline void erl_nif_env_partial_init_from_resource(ErlNifEnv *env, void * env->heap.heap_start = NULL; env->heap.heap_ptr = NULL; env->heap.heap_end = NULL; + env->heap.fragments_words = 0; env->stack_pointer = NULL; env->x[0] = term_nil(); env->x[1] = term_nil(); diff --git a/src/libAtomVM/jit.c b/src/libAtomVM/jit.c index 8d8bd5b4f8..62001d7b51 100644 --- a/src/libAtomVM/jit.c +++ b/src/libAtomVM/jit.c @@ -514,7 +514,7 @@ static Context *jit_call_ext(Context *ctx, JITState *jit_state, int offset, int ctx->e += (n_words + 1); } - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); } @@ -679,7 +679,7 @@ static bool jit_deallocate(Context *ctx, JITState *jit_state, uint32_t n_words) ctx->cp = ctx->e[n_words]; ctx->e += n_words + 1; // Hopefully, we only need x[0] - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { set_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); return false; @@ -1210,7 +1210,7 @@ static Context *jit_call_fun(Context *ctx, JITState *jit_state, int offset, term if (maybe_call_native(ctx, module_name, function_name, fun_arity, &return_value)) { PROCESS_MAYBE_TRAP_RETURN_VALUE(return_value, offset); ctx->x[0] = return_value; - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); } @@ -1718,7 +1718,7 @@ static Context *jit_apply(Context *ctx, JITState *jit_state, int offset, term mo if (maybe_call_native(ctx, module_name, function_name, arity, &native_return)) { PROCESS_MAYBE_TRAP_RETURN_VALUE(native_return, offset); ctx->x[0] = native_return; - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, ctx->x, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { return jit_raise_error(ctx, jit_state, 0, OUT_OF_MEMORY_ATOM); } diff --git a/src/libAtomVM/memory.c b/src/libAtomVM/memory.c index 7c545b9d49..b6e6ce38a5 100644 --- a/src/libAtomVM/memory.c +++ b/src/libAtomVM/memory.c @@ -73,6 +73,7 @@ void memory_init_heap_root_fragment(Heap *heap, HeapFragment *root, size_t size) heap->root = root; root->next = NULL; root->mso_list = term_nil(); + heap->fragments_words = 0; heap->heap_start = root->storage; heap->heap_ptr = heap->heap_start; heap->heap_end = heap->heap_start + size; @@ -99,10 +100,14 @@ static inline enum MemoryGCResult memory_heap_alloc_new_fragment(Heap *heap, siz HeapFragment *root_fragment = heap->root; term *old_end = heap->heap_end; term mso_list = root_fragment->mso_list; + // The old root (holding everything allocated so far) becomes a non-root + // fragment below; memory_init_heap resets the running total, so carry it. + size_t old_fragments_words = heap->fragments_words + (size_t) (heap->heap_ptr - heap->heap_start); if (UNLIKELY(memory_init_heap(heap, size) != MEMORY_GC_OK)) { TRACE("Unable to allocate memory fragment. size=%u\n", (unsigned int) size); return MEMORY_GC_ERROR_FAILED_ALLOCATION; } + heap->fragments_words = old_fragments_words; // Convert root fragment to non-root fragment. root_fragment->heap_end = old_end; // used to hold mso_list when it was the root fragment heap->root->next = root_fragment; @@ -159,7 +164,10 @@ enum MemoryGCResult memory_ensure_free_with_roots(Context *c, size_t size, size_ // Target heap size depends on: // - alloc_mode (MEMORY_FORCE_SHRINK takes precedence) // - heap growth strategy - bool should_gc = free_space < size || (alloc_mode == MEMORY_FORCE_SHRINK) || c->heap.root->next != NULL; + // Heap fragments (literals decoded from the module literal table, NIF + // results, received messages) are folded in only once they are large + // (see memory_heap_fragments_need_gc), otherwise at the next natural GC. + bool should_gc = free_space < size || (alloc_mode == MEMORY_FORCE_SHRINK) || memory_heap_fragments_need_gc(&c->heap); size_t memory_size = 0; if (!should_gc) { switch (c->heap_growth_strategy) { @@ -906,6 +914,7 @@ HOT_FUNC static term memory_shallow_copy_term(HeapFragment *old_fragment, term t void memory_heap_append_fragment(Heap *heap, HeapFragment *fragment, term mso_list) { + heap->fragments_words += memory_heap_fragment_memory_size(fragment); // The fragment we are appending may have next fragments // So we take our current next and we add it to the tail of the passed list if (heap->root->next) { diff --git a/src/libAtomVM/memory.h b/src/libAtomVM/memory.h index e7317bebd4..c5f2b27250 100644 --- a/src/libAtomVM/memory.h +++ b/src/libAtomVM/memory.h @@ -85,6 +85,10 @@ struct Heap term *heap_start; term *heap_ptr; term *heap_end; + // Running total of the words held in the fragments chained off root + // (root->next...), so heap size and the fold-in decision are O(1) per + // allocation instead of a chain walk. + size_t fragments_words; }; #ifndef TYPEDEF_HEAP @@ -176,11 +180,34 @@ static inline size_t memory_heap_youngest_size(const Heap *heap) */ static inline size_t memory_heap_memory_size(const Heap *heap) { - size_t result = memory_heap_youngest_size(heap); - if (heap->root->next) { - result += memory_heap_fragment_memory_size(heap->root->next); + // Called on every allocation under the fibonacci growth policy: keep it + // O(1) via the running fragment total. + return memory_heap_youngest_size(heap) + heap->fragments_words; +} + +/** + * @brief Whether the heap's fragments are worth folding in right now. + * + * @details Fragments come from decoded literals (every use of a compound + * literal), NIF results and received messages. Any fragment at all used to + * force a shrinking GC at the next return / NIF call / allocation; for code + * that references compound literals in nearly every function (Gleam- and + * Elixir-compiled code in particular) that was a full copying collection + * every few instructions. Fragments are valid heap memory the collector + * copies out of like anything else, so they are folded in only once they are + * large relative to the heap or in absolute terms; otherwise they wait for + * the next natural collection. + * @param heap the heap + * @return true if a collection should be forced to merge the fragments + */ +static inline bool memory_heap_fragments_need_gc(const Heap *heap) +{ + if (heap->root->next == NULL) { + return false; } - return result; + size_t frag_size = heap->fragments_words; + size_t young_size = memory_heap_youngest_size(heap); + return frag_size > young_size / 4 || frag_size > 65536; } /** diff --git a/src/libAtomVM/opcodesswitch.h b/src/libAtomVM/opcodesswitch.h index 889ff319a8..e49f7d5e9f 100644 --- a/src/libAtomVM/opcodesswitch.h +++ b/src/libAtomVM/opcodesswitch.h @@ -1106,7 +1106,7 @@ static inline ModuleNativeEntryPoint do_return_native(Module *mod, Context *ctx) if (maybe_call_native(ctx, module_name, function_name, fun_arity, &return_value)) { \ PROCESS_MAYBE_TRAP_RETURN_VALUE(return_value); \ x_regs[0] = return_value; \ - if (ctx->heap.root->next) { \ + if (memory_heap_fragments_need_gc(&ctx->heap)) { \ if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { \ RAISE_ERROR(OUT_OF_MEMORY_ATOM); \ } \ @@ -1919,7 +1919,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term return_value = nif->nif_ptr(ctx, arity, x_regs); PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_PC_INDEX_ARITY(return_value, orig_pc, mod, index, arity); x_regs[0] = return_value; - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -2044,7 +2044,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->cp = ctx->e[n_words]; ctx->e += (n_words + 1); - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -2333,7 +2333,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->e += n_words + 1; DEBUG_DUMP_STACK(ctx); // Hopefully, we only need x[0] - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } @@ -2617,11 +2617,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_eq_exact/3, label=%" PRIu32 ", arg1=%" TERM_X_FMT ", arg2=%" TERM_X_FMT "\n", label, arg1, arg2); - TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global); - if (result & (TermLessThan | TermGreaterThan)) { - pc = mod->labels[label]; - } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { - RAISE_ERROR(OUT_OF_MEMORY_ATOM); + bool fast_equal; + if (term_exact_eq_fast(arg1, arg2, &fast_equal)) { + if (!fast_equal) { + pc = mod->labels[label]; + } + } else { + TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global); + if (result & (TermLessThan | TermGreaterThan)) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } } break; @@ -2637,11 +2644,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_not_eq_exact/3, label=%" PRIu32 ", arg1=%" TERM_X_FMT ", arg2=%" TERM_X_FMT "\n", label, arg1, arg2); - TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global); - if (result == TermEquals) { - pc = mod->labels[label]; - } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { - RAISE_ERROR(OUT_OF_MEMORY_ATOM); + bool fast_equal; + if (term_exact_eq_fast(arg1, arg2, &fast_equal)) { + if (fast_equal) { + pc = mod->labels[label]; + } + } else { + TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global); + if (result == TermEquals) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } } break; @@ -2865,12 +2879,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DECODE_LABEL(jmp_label, pc) if (!jump_to_address) { - TermCompareResult result = term_compare( - src_value, cmp_value, TermCompareExact, ctx->global); - if (result == TermEquals) { - jump_to_address = mod->labels[jmp_label]; - } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { - RAISE_ERROR(OUT_OF_MEMORY_ATOM); + bool fast_equal; + if (term_exact_eq_fast(src_value, cmp_value, &fast_equal)) { + if (fast_equal) { + jump_to_address = mod->labels[jmp_label]; + } + } else { + TermCompareResult result = term_compare( + src_value, cmp_value, TermCompareExact, ctx->global); + if (result == TermEquals) { + jump_to_address = mod->labels[jmp_label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } } } } @@ -3135,7 +3156,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) PROCESS_MAYBE_TRAP_RETURN_VALUE_LAST(return_value); x_regs[0] = return_value; - if (ctx->heap.root->next) { + if (memory_heap_fragments_need_gc(&ctx->heap)) { if (UNLIKELY(memory_ensure_free_with_roots(ctx, 0, 1, x_regs, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } diff --git a/src/libAtomVM/scheduler.c b/src/libAtomVM/scheduler.c index a856d60012..6dc5275556 100644 --- a/src/libAtomVM/scheduler.c +++ b/src/libAtomVM/scheduler.c @@ -296,7 +296,7 @@ Context *scheduler_run(GlobalContext *global) if (result->native_handler(result) == NativeContinue) { // If native handler has memory fragments, garbage collect // them - if (result->heap.root->next) { + if (memory_heap_fragments_need_gc(&result->heap)) { if (UNLIKELY(memory_ensure_free_opt(result, 0, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { fprintf(stderr, "Out of memory error in native handler\n"); AVM_ABORT(); diff --git a/src/libAtomVM/term.h b/src/libAtomVM/term.h index df1d411354..bc6c61a195 100644 --- a/src/libAtomVM/term.h +++ b/src/libAtomVM/term.h @@ -439,6 +439,34 @@ static inline bool term_is_boxed(term t) return ((t & TERM_PRIMARY_MASK) == TERM_PRIMARY_BOXED); } +/** + * @brief Exact-equality fast path for immediates. + * + * @details Two terms with the same bits are `=:=` equal. Two immediate terms + * (atoms, small integers, nil, local pids, ...) with different bits are never + * `=:=` equal, because immediates are canonical (a small integer is never + * boxed). Only when at least one side is boxed or a list must the caller fall + * back to term_compare, which for two different atoms fetches both names from + * the atom table and compares them: previously the dominant cost of every + * `case` on atoms (OP_SELECT_VAL) in Gleam-compiled code. + * @param a first term + * @param b second term + * @param equal set to the answer when the function returns true + * @return true if the answer is decided (in *equal), false if term_compare is needed + */ +static inline bool term_exact_eq_fast(term a, term b, bool *equal) +{ + if (a == b) { + *equal = true; + return true; + } + if (((a & TERM_PRIMARY_MASK) == TERM_PRIMARY_IMMED) && ((b & TERM_PRIMARY_MASK) == TERM_PRIMARY_IMMED)) { + *equal = false; + return true; + } + return false; +} + /** * @brief Returns size of a boxed term from its header *