From 0d1625e553111509dcd28ab259a5e3d96abd2856 Mon Sep 17 00:00:00 2001 From: SirOlaf <34164198+SirOlaf@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:09:52 +0200 Subject: [PATCH 1/5] Dual queue --- lib/system.nim | 16 +++- lib/system/alloc.nim | 195 ++++++++++++++++++++++++++++---------- lib/system/arc.nim | 18 ++-- lib/system/threadimpl.nim | 11 +++ 4 files changed, 180 insertions(+), 60 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 313f90969cbc0..1c55af7d663a1 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1154,7 +1154,19 @@ template sysAssert(cond: bool, msg: string) = cstderr.rawWrite "\n" rawQuit 1 -const hasAlloc = (hostOS != "standalone" or not defined(nogc)) and not defined(nimscript) +const + hasAlloc = (hostOS != "standalone" or not defined(nogc)) and not defined(nimscript) + hasDefaultAllocator = + hasAlloc and + not (defined(useNimRtl) or defined(useMalloc) or defined(gcRegions) or + defined(nogc) or defined(boehmgc) or defined(gogc)) + hasThreadLocalAllocator = + hasDefaultAllocator and hasThreadSupport and defined(gcDestructors) + +when hasThreadLocalAllocator: + # threadimpl is included before mmdisp provides these implementations. + proc initThreadAllocator() {.gcsafe, raises: [].} + proc releaseThreadAllocator() {.gcsafe, raises: [].} when notJSnotNims and hasAlloc and not defined(nimSeqsV2): proc addChar(s: NimString, c: char): NimString {.compilerproc, gcsafe.} @@ -2425,6 +2437,8 @@ when notJSnotNims and hasAlloc: {.push profiler: off.} include "system/mmdisp" {.pop.} + when hasThreadLocalAllocator and not defined(createNimRtl): + initThreadAllocator() {.push stackTrace: off, profiler: off.} when not defined(nimSeqsV2): include "system/sysstr" diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 880fdeb388937..a000e16572394 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -40,7 +40,7 @@ template track(op, address, size) = # # A deallocation of a small pointer then looks like this #[ - dealloc -> rawDealloc -> chunk.owner == addr(a) --------------> This thread owns the chunk ------> The current chunk is active -> Chunk is completely unused -----> Chunk references no foreign cells + dealloc -> rawDealloc -> chunk.owner == regionOwner(a) -------> This thread owns the chunk ------> The current chunk is active -> Chunk is completely unused -----> Chunk references no foreign cells | | (Add cell into the current chunk) | Return the current chunk back to tlsf | | | | v v v v @@ -63,6 +63,7 @@ const # size of chunks in last matrix bin MaxBigChunkSize = int(1'i32 shl MaxFli - 1'i32 shl (MaxFli-MaxLog2Sli-1)) HugeChunkSize = MaxBigChunkSize + 1 + usesRegionHandles = hasThreadSupport and defined(gcDestructors) type PTrunk = ptr Trunk @@ -112,11 +113,19 @@ type PChunk = ptr BaseChunk PBigChunk = ptr BigChunk PSmallChunk = ptr SmallChunk + SharedFreeLists = array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell] + RegionHandle = object + # Permanent chunk-owner identity and home of the remote-free queues. + sharedFreeLists: SharedFreeLists + sharedFreeListBigChunks: PBigChunk BaseChunk {.pure, inheritable.} = object prevSize: int # size of previous chunk; for coalescing # 0th bit == 1 if 'used size: int # if < PageSize it is a small chunk - owner: ptr MemRegion + when usesRegionHandles: + owner: ptr RegionHandle + else: + owner: ptr MemRegion SmallChunk = object of BaseChunk next, prev: PSmallChunk # chunks of the same size @@ -145,14 +154,16 @@ type next: ptr HeapLinks MemRegion = object + when usesRegionHandles: + regionHandle: ptr RegionHandle when not defined(gcDestructors): minLargeObj, maxLargeObj: int freeSmallChunks: array[0..max(1, SmallChunkSize div MemAlign-1), PSmallChunk] # List of available chunks per size class. Only one is expected to be active per class. when defined(gcDestructors): - sharedFreeLists: array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell] - # When a thread frees a pointer it did not create, it must not adjust the counters. - # Instead, the cell is placed here and deferred until the next allocation. + sharedFreeLists: SharedFreeLists + # Used directly without threads. Threaded builds use RegionHandle but + # retain this 2 KiB spacer: removing it regresses 2-4 KiB allocations. flBitmap: uint32 slBitmap: array[RealFli, uint32] matrix: array[RealFli, array[MaxSli, PBigChunk]] @@ -160,7 +171,7 @@ type currMem, maxMem, freeMem, occ: int # memory sizes (allocated from OS) lastSize: int # needed for the case that OS gives us pages linearly when defined(gcDestructors): - sharedFreeListBigChunks: PBigChunk # make no attempt at avoiding false sharing for now for this object field + sharedFreeListBigChunks: PBigChunk # private pending list with threads; shared queue otherwise chunkStarts: IntSet when not defined(gcDestructors): @@ -173,9 +184,19 @@ type when defined(nimTypeNames): allocCounter, deallocCounter: int + PooledRegion = object + region: MemRegion + next: ptr PooledRegion + template smallChunkOverhead(): untyped = sizeof(SmallChunk) template bigChunkOverhead(): untyped = sizeof(BigChunk) +template regionOwner(a: var MemRegion): untyped = + when usesRegionHandles: + a.regionHandle + else: + addr a + when hasThreadSupport: template loada(x: untyped): untyped = atomicLoadN(unsafeAddr x, ATOMIC_RELAXED) template storea(x, y: untyped) = atomicStoreN(unsafeAddr x, y, ATOMIC_RELAXED) @@ -502,6 +523,51 @@ proc pageAddr(p: pointer): PChunk {.inline.} = result = cast[PChunk](cast[int](p) and not PageMask) #sysAssert(Contains(allocator.chunkStarts, pageIndex(result))) +when hasThreadLocalAllocator: + var + regionPool: ptr PooledRegion + regionPoolLock: SysLock + initSysLock(regionPoolLock) + + proc moveMemRegion(dest, source: ptr MemRegion) {.inline.} = + # MemRegion owns only raw allocator state, so transfer it bitwise and + # clear the source to leave exactly one owner. + copyMem(dest, source, sizeof(MemRegion)) + zeroMem(source, sizeof(MemRegion)) + + proc acquireMemRegion(a: var MemRegion) {.raises: [], gcsafe.} = + if a.regionHandle != nil: + return + + acquireSys(regionPoolLock) + let pooled = regionPool + if pooled != nil: + regionPool = pooled.next + releaseSys(regionPoolLock) + + if pooled == nil: + let handle = cast[ptr RegionHandle](c_malloc(csize_t sizeof(RegionHandle))) + if handle == nil: + raiseOutOfMem() + zeroMem(handle, sizeof(RegionHandle)) + a.regionHandle = handle + else: + moveMemRegion(addr a, addr pooled.region) + c_free(pooled) + + proc releaseMemRegion(a: var MemRegion) {.raises: [], gcsafe.} = + if a.regionHandle == nil: + return + let pooled = cast[ptr PooledRegion](c_malloc(csize_t sizeof(PooledRegion))) + if pooled == nil: + raiseOutOfMem() + moveMemRegion(addr pooled.region, addr a) + + acquireSys(regionPoolLock) + pooled.next = regionPool + regionPool = pooled + releaseSys(regionPoolLock) + when false: proc writeFreeList(a: MemRegion) = var it = a.freeChunksList @@ -618,7 +684,7 @@ proc splitChunk2(a: var MemRegion, c: PBigChunk, size: int): PBigChunk = result.prev = nil # size and not used: result.prevSize = size - result.owner = addr a + result.owner = regionOwner(a) sysAssert((size and 1) == 0, "splitChunk 2") sysAssert((size and PageMask) == 0, "splitChunk: size is not a multiple of the PageSize") @@ -686,7 +752,7 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # if we over allocated split the chunk: if result.size > size: splitChunk(a, result, size) - result.owner = addr a + result.owner = regionOwner(a) else: removeChunkFromMatrix2(a, result, fl, sl) if result.size >= size + PageSize: @@ -694,7 +760,7 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk = # set 'used' to true: result.prevSize = 1 track("setUsedToFalse", addr result.size, sizeof(int)) - sysAssert result.owner == addr a, "getBigChunk: No owner set!" + sysAssert result.owner == regionOwner(a), "getBigChunk: No owner set!" incl(a, a.chunkStarts, pageIndex(result)) dec(a.freeMem, size) @@ -710,7 +776,7 @@ proc getHugeChunk(a: var MemRegion; size: int): PBigChunk = result.size = size # set 'used' to true: result.prevSize = 1 - result.owner = addr a + result.owner = regionOwner(a) incl(a, a.chunkStarts, pageIndex(result)) proc freeHugeChunk(a: var MemRegion; c: PBigChunk) = @@ -791,7 +857,7 @@ proc deallocBigChunk(a: var MemRegion, c: PBigChunk) = when defined(gcDestructors): template atomicPrepend(head, elem: untyped) = # see also https://en.cppreference.com/w/cpp/atomic/atomic_compare_exchange - when hasThreadSupport: + when usesRegionHandles: while true: elem.next.storea head.loada if atomicCompareExchangeN(addr head, addr elem.next, elem, weak = true, ATOMIC_RELEASE, ATOMIC_RELAXED): @@ -800,30 +866,39 @@ when defined(gcDestructors): elem.next.storea head.loada head.storea elem - proc addToSharedFreeListBigChunks(a: var MemRegion; c: PBigChunk) {.inline.} = - sysAssert c.next == nil, "c.next pointer must be nil" - atomicPrepend a.sharedFreeListBigChunks, c + when usesRegionHandles: + proc addToSharedFreeListBigChunks(handle: ptr RegionHandle; + c: PBigChunk) {.inline.} = + sysAssert c.next == nil, "c.next pointer must be nil" + atomicPrepend handle.sharedFreeListBigChunks, c + else: + proc addToSharedFreeListBigChunks(a: var MemRegion; + c: PBigChunk) {.inline.} = + sysAssert c.next == nil, "c.next pointer must be nil" + atomicPrepend a.sharedFreeListBigChunks, c proc takeFromSharedFreeListBigChunks(a: var MemRegion): PBigChunk {.inline.} = - when hasThreadSupport: - while true: - result = atomicLoadN(addr a.sharedFreeListBigChunks, ATOMIC_ACQUIRE) - if result == nil: - break - let next = result.next.loada - var expected = result - if atomicCompareExchangeN(addr a.sharedFreeListBigChunks, addr expected, next, - weak = true, ATOMIC_ACQUIRE, ATOMIC_RELAXED): - result.next.storea nil - break - else: - result = a.sharedFreeListBigChunks - if result != nil: - a.sharedFreeListBigChunks = result.next - result.next = nil - - proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; size: int) {.inline.} = - atomicPrepend c.owner.sharedFreeLists[size], f + when usesRegionHandles: + if a.sharedFreeListBigChunks == nil: + let sharedHead = addr a.regionHandle.sharedFreeListBigChunks + # Detach a batch from the stable remote inbox. The embedded MemRegion + # field is now a private pending list and moves with the region. + if atomicLoadN(sharedHead, ATOMIC_RELAXED) != nil: + a.sharedFreeListBigChunks = atomicExchangeN(sharedHead, nil, + ATOMIC_ACQUIRE) + result = a.sharedFreeListBigChunks + if result != nil: + a.sharedFreeListBigChunks = result.next + result.next = nil + + when usesRegionHandles: + proc addToSharedFreeList(handle: ptr RegionHandle; f: ptr FreeCell; + size: int) {.inline.} = + atomicPrepend handle.sharedFreeLists[size], f + else: + proc addToSharedFreeList(c: PSmallChunk; f: ptr FreeCell; + size: int) {.inline.} = + atomicPrepend c.owner.sharedFreeLists[size], f const MaxSteps = 20 @@ -846,9 +921,8 @@ when defined(gcDestructors): dec(a.occ, total) proc freeDeferredObjects(a: var MemRegion) = - # Pop only as many nodes as we can process. Detaching the entire list and - # re-enqueuing its unprocessed tail through atomicPrepend would overwrite - # that tail's next pointer and lose the rest of the list. + # Bound the work per allocation. With threads, takeFromSharedFreeListBigChunks + # detaches the shared stack into the region's private pending list first. for _ in 0..MaxSteps: let it = takeFromSharedFreeListBigChunks(a) if it == nil: break @@ -892,17 +966,20 @@ proc rawAlloc(a: var MemRegion, requestedSize: int, alignment: int = 0): pointer if size + alignOff <= SmallChunkSize-smallChunkOverhead(): template fetchSharedCells(tc: PSmallChunk) = - # Consumes cells from (potentially) foreign threads from `a.sharedFreeLists[s]` + # Consume cells freed by potentially foreign threads. when defined(gcDestructors): if tc.freeList == nil: - when hasThreadSupport: - # Steal the entire list from `sharedFreeList`: - tc.freeList = atomicExchangeN(addr a.sharedFreeLists[s], nil, ATOMIC_RELAXED) + when usesRegionHandles: + let sharedHead = addr tc.owner.sharedFreeLists[s] + # The owner is the only consumer, so once it observes a non-empty + # stack no other thread can make it empty before the exchange. + if atomicLoadN(sharedHead, ATOMIC_RELAXED) != nil: + tc.freeList = atomicExchangeN(sharedHead, nil, ATOMIC_ACQUIRE) else: tc.freeList = a.sharedFreeLists[s] a.sharedFreeLists[s] = nil - # if `tc.freeList` isn't nil, `tc` will gain capacity. - # We must calculate how much it gained and how many foreign cells are included. + # If `tc.freeList` isn't nil, `tc` gains capacity. Calculate how + # much it gained and how many foreign cells are included. compensateCounters(a, tc, size) # allocate a small block: for small chunks, we use only its next pointer @@ -921,11 +998,11 @@ proc rawAlloc(a: var MemRegion, requestedSize: int, alignment: int = 0): pointer c.size = size c.acc = (alignOff + size).uint32 c.free = SmallChunkSize - smallChunkOverhead() - alignOff.int32 - size.int32 - sysAssert c.owner == addr(a), "rawAlloc: No owner set!" + sysAssert c.owner == regionOwner(a), "rawAlloc: No owner set!" c.next = nil c.prev = nil - # Shared cells are fetched here in case `c.size * 2 >= SmallChunkSize - smallChunkOverhead()`. - # For those single cell chunks, we would otherwise have to allocate a new one almost every time. + # Fetch deferred cells here for single-cell chunks; otherwise every + # allocation of that size would tend to allocate a new chunk. fetchSharedCells(c) if c.free >= size: # Because removals from `a.freeSmallChunks[s]` only happen in the other alloc branch and during dealloc, @@ -963,9 +1040,8 @@ proc rawAlloc(a: var MemRegion, requestedSize: int, alignment: int = 0): pointer dec(c.free, size) sysAssert((cast[int](result) and (MemAlign-1)) == 0, "rawAlloc 9") sysAssert(allocInv(a), "rawAlloc: end c != nil") - # We fetch deferred cells *after* advancing `c.freeList`/`acc` to adjust `c.free`. - # If after the adjustment it turns out there's free cells available, - # the chunk stays in `a.freeSmallChunks[s]` and the need for a new chunk is delayed. + # Fetch after advancing `freeList`/`acc` so `c.free` can be adjusted. If + # cells arrived, keep this chunk active instead of allocating another. fetchSharedCells(c) sysAssert(allocInv(a), "rawAlloc: before c.free < size") if c.free < size: @@ -1030,7 +1106,8 @@ proc rawDealloc(a: var MemRegion, p: pointer) = # ^ We might access thread foreign storage here. # The other thread cannot possibly free this block as it's still alive. var f = cast[ptr FreeCell](p) - if c.owner == addr(a): + let owner = c.owner + if owner == regionOwner(a): # We own the block, there is no foreign thread involved. dec a.occ, s untrackSize(s) @@ -1093,7 +1170,10 @@ proc rawDealloc(a: var MemRegion, p: pointer) = when logAlloc: cprintf("dealloc(pointer_%p) # SMALL FROM %p CALLER %p\n", p, c.owner, addr(a)) when defined(gcDestructors): - addToSharedFreeList(c, f, s div MemAlign) + when usesRegionHandles: + addToSharedFreeList(owner, f, s div MemAlign) + else: + addToSharedFreeList(c, f, s div MemAlign) sysAssert(((cast[int](p) and PageMask) - smallChunkOverhead() - c.chunkAlignOff) %% s == 0, "rawDealloc 2") else: @@ -1101,10 +1181,14 @@ proc rawDealloc(a: var MemRegion, p: pointer) = when overwriteFree: nimSetMem(p, -1'i32, c.size -% bigChunkOverhead()) when logAlloc: cprintf("dealloc(pointer_%p) # BIG %p\n", p, c.owner) when defined(gcDestructors): - if c.owner == addr(a): + let owner = c.owner + if owner == regionOwner(a): deallocBigChunk(a, cast[PBigChunk](c)) else: - addToSharedFreeListBigChunks(c.owner[], cast[PBigChunk](c)) + when usesRegionHandles: + addToSharedFreeListBigChunks(owner, cast[PBigChunk](c)) + else: + addToSharedFreeListBigChunks(owner[], cast[PBigChunk](c)) else: deallocBigChunk(a, cast[PBigChunk](c)) @@ -1263,6 +1347,13 @@ when defined(nimTypeNames): template instantiateForRegion(allocator: untyped) {.dirty.} = {.push stackTrace: off.} + when hasThreadLocalAllocator: + proc initThreadAllocator() {.gcsafe, raises: [].} = + acquireMemRegion(allocator) + + proc releaseThreadAllocator() {.gcsafe, raises: [].} = + releaseMemRegion(allocator) + when defined(nimFulldebug): proc interiorAllocatedPtr*(p: pointer): pointer = result = interiorAllocatedPtr(allocator, p) diff --git a/lib/system/arc.nim b/lib/system/arc.nim index d380aa621db02..114b671bd797b 100644 --- a/lib/system/arc.nim +++ b/lib/system/arc.nim @@ -283,13 +283,17 @@ when not (defined(gcOrc) or defined(gcYrc)): ## Forces a full garbage collection pass. With `--mm:arc` a nop. discard -template setupForeignThreadGc* = - ## With `--mm:arc` a nop. - discard - -template tearDownForeignThreadGc* = - ## With `--mm:arc` a nop. - discard +when not hasThreadSupport: + template setupForeignThreadGc* = discard + template tearDownForeignThreadGc* = discard +elif emulatedThreadVars: + template setupForeignThreadGc* = + {.error: "setupForeignThreadGc is available only when ``--threads:on`` and ``--tlsEmulation:off`` are used".} + template tearDownForeignThreadGc* = + {.error: "tearDownForeignThreadGc is available only when ``--threads:on`` and ``--tlsEmulation:off`` are used".} +elif not hasThreadLocalAllocator: + template setupForeignThreadGc* = discard + template tearDownForeignThreadGc* = discard proc isObjDisplayCheck(source: PNimTypeV2, targetDepth: int16, token: uint32): bool {.compilerRtl, inl.} = result = targetDepth <= source.depth and source.display[targetDepth] == token diff --git a/lib/system/threadimpl.nim b/lib/system/threadimpl.nim index e35378db0e03e..62d54ef6d84a8 100644 --- a/lib/system/threadimpl.nim +++ b/lib/system/threadimpl.nim @@ -19,6 +19,13 @@ when not defined(useNimRtl): threadType = ThreadType.NimThread +when hasThreadLocalAllocator and not emulatedThreadVars: + proc setupForeignThreadGc*() {.gcsafe, raises: [].} = + initThreadAllocator() + + proc tearDownForeignThreadGc*() {.gcsafe, raises: [].} = + releaseThreadAllocator() + when defined(gcDestructors): proc deallocThreadStorage(p: pointer) = c_free(p) else: @@ -83,6 +90,8 @@ else: deallocThreadStorage(thrd.rawStack) proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = + when hasThreadLocalAllocator: + initThreadAllocator() when defined(boehmgc): boehmGC_call_with_stack_base(threadProcWrapDispatch[TArg], thrd) elif not defined(nogc) and not defined(gogc) and not defined(gcRegions) and not usesDestructors: @@ -97,6 +106,8 @@ proc threadProcWrapStackFrame[TArg](thrd: ptr Thread[TArg]) {.raises: [].} = when declared(deallocOsPages): deallocOsPages() else: threadProcWrapDispatch(thrd) + when hasThreadLocalAllocator: + releaseThreadAllocator() template nimThreadProcWrapperBody*(closure: untyped): untyped = var thrd = cast[ptr Thread[TArg]](closure) From e7243ef3e4c6cdae51d666d7f7827e35e294ad61 Mon Sep 17 00:00:00 2001 From: SirOlaf <34164198+SirOlaf@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:17:22 +0200 Subject: [PATCH 2/5] Tests --- tests/threads/tthreadallocatorforeignpool.nim | 59 ++++++++++++ tests/threads/tthreadallocatorhandoffrace.nim | 64 +++++++++++++ tests/threads/tthreadallocatorpool.nim | 92 +++++++++++++++++++ tests/threads/tthreadallocatorpoolrace.nim | 56 +++++++++++ 4 files changed, 271 insertions(+) create mode 100644 tests/threads/tthreadallocatorforeignpool.nim create mode 100644 tests/threads/tthreadallocatorhandoffrace.nim create mode 100644 tests/threads/tthreadallocatorpool.nim create mode 100644 tests/threads/tthreadallocatorpoolrace.nim diff --git a/tests/threads/tthreadallocatorforeignpool.nim b/tests/threads/tthreadallocatorforeignpool.nim new file mode 100644 index 0000000000000..ad46c74fea333 --- /dev/null +++ b/tests/threads/tthreadallocatorforeignpool.nim @@ -0,0 +1,59 @@ +discard """ + matrix: "--mm:arc --threads:on --tlsEmulation:off; --mm:orc --threads:on --tlsEmulation:off" + disabled: "windows" + output: "ok" + timeout: "30" +""" + +import std/posix + +var + escaped: pointer + reused: pointer + +proc allocateOnForeignThread(_: pointer): pointer {.noconv.} = + setupForeignThreadGc() + escaped = allocShared(96) + cast[ptr int](escaped)[] = 73 + tearDownForeignThreadGc() + result = nil + +proc reuseOnForeignThread(_: pointer): pointer {.noconv.} = + setupForeignThreadGc() + doAssert cast[ptr int](escaped)[] == 73 + deallocShared(escaped) + reused = allocShared(96) + doAssert reused == escaped + deallocShared(reused) + tearDownForeignThreadGc() + result = nil + +proc consumeDeferredFree(_: pointer): pointer {.noconv.} = + setupForeignThreadGc() + let first = allocShared(96) + let second = allocShared(96) + # The first allocation advances the active chunk and collects its deferred + # foreign frees. The next allocation reuses the remotely returned cell. + doAssert second == escaped + deallocShared(first) + deallocShared(second) + tearDownForeignThreadGc() + result = nil + +proc run(worker: proc(_: pointer): pointer {.noconv.}) = + var thread: Pthread + doAssert pthread_create(addr thread, nil, worker, nil) == 0 + doAssert pthread_join(thread, nil) == 0 + +# setup/teardown is the checkout/return boundary. A distinct native thread can +# safely inherit the allocator even while one of its allocations is still live. +run(allocateOnForeignThread) +run(reuseOnForeignThread) + +# A free that arrives while the allocator is idle is queued on its handle and +# consumed after that allocator is handed to another foreign thread. +run(allocateOnForeignThread) +deallocShared(escaped) +run(consumeDeferredFree) + +echo "ok" diff --git a/tests/threads/tthreadallocatorhandoffrace.nim b/tests/threads/tthreadallocatorhandoffrace.nim new file mode 100644 index 0000000000000..3a7803e8f7be2 --- /dev/null +++ b/tests/threads/tthreadallocatorhandoffrace.nim @@ -0,0 +1,64 @@ +discard """ + matrix: "--mm:arc --threads:on; --mm:orc --threads:on" + output: "ok" + timeout: "30" +""" + +import std/[atomics, typedthreads] + +const + pointerCount = 512 + drainCount = 2048 + iterations {.intdefine.} = 200 + sizes = [16, 64, 4000, 4096, 8192] + +var + pointers: array[pointerCount, pointer] + mayExit: Atomic[bool] + +proc owner() {.thread.} = + for i in 0.. Date: Sun, 2 Aug 2026 10:56:38 +0200 Subject: [PATCH 3/5] Remove old artifact --- lib/system/alloc.nim | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index a000e16572394..a5850f3c12380 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -114,10 +114,6 @@ type PBigChunk = ptr BigChunk PSmallChunk = ptr SmallChunk SharedFreeLists = array[0..max(1, SmallChunkSize div MemAlign-1), ptr FreeCell] - RegionHandle = object - # Permanent chunk-owner identity and home of the remote-free queues. - sharedFreeLists: SharedFreeLists - sharedFreeListBigChunks: PBigChunk BaseChunk {.pure, inheritable.} = object prevSize: int # size of previous chunk; for coalescing # 0th bit == 1 if 'used @@ -184,9 +180,14 @@ type when defined(nimTypeNames): allocCounter, deallocCounter: int - PooledRegion = object + RegionHandle = object + # Permanent chunk-owner identity and home of the remote-free queues. + sharedFreeLists: SharedFreeLists + sharedFreeListBigChunks: PBigChunk + # Keep the movable allocator state with its permanent owner while the + # owning thread is retired. region: MemRegion - next: ptr PooledRegion + next: ptr RegionHandle template smallChunkOverhead(): untyped = sizeof(SmallChunk) template bigChunkOverhead(): untyped = sizeof(BigChunk) @@ -525,7 +526,7 @@ proc pageAddr(p: pointer): PChunk {.inline.} = when hasThreadLocalAllocator: var - regionPool: ptr PooledRegion + regionPool: ptr RegionHandle regionPoolLock: SysLock initSysLock(regionPoolLock) @@ -540,32 +541,29 @@ when hasThreadLocalAllocator: return acquireSys(regionPoolLock) - let pooled = regionPool - if pooled != nil: - regionPool = pooled.next + let handle = regionPool + if handle != nil: + regionPool = handle.next releaseSys(regionPoolLock) - if pooled == nil: - let handle = cast[ptr RegionHandle](c_malloc(csize_t sizeof(RegionHandle))) - if handle == nil: + if handle == nil: + let newHandle = cast[ptr RegionHandle](c_malloc(csize_t sizeof(RegionHandle))) + if newHandle == nil: raiseOutOfMem() - zeroMem(handle, sizeof(RegionHandle)) - a.regionHandle = handle + zeroMem(newHandle, sizeof(RegionHandle)) + a.regionHandle = newHandle else: - moveMemRegion(addr a, addr pooled.region) - c_free(pooled) + moveMemRegion(addr a, addr handle.region) proc releaseMemRegion(a: var MemRegion) {.raises: [], gcsafe.} = if a.regionHandle == nil: return - let pooled = cast[ptr PooledRegion](c_malloc(csize_t sizeof(PooledRegion))) - if pooled == nil: - raiseOutOfMem() - moveMemRegion(addr pooled.region, addr a) + let handle = a.regionHandle + moveMemRegion(addr handle.region, addr a) acquireSys(regionPoolLock) - pooled.next = regionPool - regionPool = pooled + handle.next = regionPool + regionPool = handle releaseSys(regionPoolLock) when false: From 861936849a0c9b335c6bcce948d169f7a330a366 Mon Sep 17 00:00:00 2001 From: SirOlaf <34164198+SirOlaf@users.noreply.github.com> Date: Sun, 2 Aug 2026 12:47:41 +0200 Subject: [PATCH 4/5] Rtl needs the allocator too --- lib/system.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/system.nim b/lib/system.nim index 1c55af7d663a1..d4896bdc4ad61 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -2437,7 +2437,7 @@ when notJSnotNims and hasAlloc: {.push profiler: off.} include "system/mmdisp" {.pop.} - when hasThreadLocalAllocator and not defined(createNimRtl): + when hasThreadLocalAllocator: initThreadAllocator() {.push stackTrace: off, profiler: off.} when not defined(nimSeqsV2): From 8351874476e9f2c3004f016d14c03875b47a6b33 Mon Sep 17 00:00:00 2001 From: SirOlaf <34164198+SirOlaf@users.noreply.github.com> Date: Wed, 5 Aug 2026 11:40:37 +0200 Subject: [PATCH 5/5] Use osAllocPages for handles --- lib/system/alloc.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index a5850f3c12380..21708b660bf41 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -547,9 +547,10 @@ when hasThreadLocalAllocator: releaseSys(regionPoolLock) if handle == nil: - let newHandle = cast[ptr RegionHandle](c_malloc(csize_t sizeof(RegionHandle))) - if newHandle == nil: - raiseOutOfMem() + # RegionHandle is larger than llAlloc's one-page metadata slabs and is + # retained independently of any checked-out MemRegion. + let handleSize = roundup(sizeof(RegionHandle), PageSize) + let newHandle = cast[ptr RegionHandle](osAllocPages(handleSize)) zeroMem(newHandle, sizeof(RegionHandle)) a.regionHandle = newHandle else: