diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 10c4537d7..8d8482597 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -574,7 +574,7 @@ jobs: continue-on-error: true steps: - uses: actions/checkout@v6 - - name: netbsd + - name: dragonflybsd uses: vmactions/dragonflybsd-vm@v1 with: copyback: false diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e2ed298c..79fad5349 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,8 @@ include(CMakePackageConfigHelpers) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) +set(QTHREADS_BUILD_BENCHMARKS OFF CACHE BOOL "Whether or not to build the qthreads benchmarks.") + add_subdirectory(src) if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) diff --git a/include/qt_atomic_wait.h b/include/qt_atomic_wait.h index 49b0c1917..142e56620 100644 --- a/include/qt_atomic_wait.h +++ b/include/qt_atomic_wait.h @@ -1,9 +1,18 @@ #ifndef QT_ATOMIC_WAIT_H #define QT_ATOMIC_WAIT_H +#include +#include + +#ifdef __cplusplus +#include +#include +#include +#else #include #include #include +#endif #include "qt_asserts.h" #include "qt_os.h" @@ -16,16 +25,29 @@ // Linux only has 32-bit futexes so that's the only size that's possible to // standardize. +#ifdef __cplusplus +#define qt_atomic_wait_t std::atomic +#define qt_atomic_wait_empty 0u +#define qt_atomic_wait_full UINT32_MAX +#define qt_atomic_wait_set_empty(a) \ + ((a)->store(qt_atomic_wait_empty, std::memory_order_relaxed)) +#define qt_atomic_wait_set_full(a) \ + ((a)->store(qt_atomic_wait_full, std::memory_order_relaxed)) +#define qt_atomic_wait_load(a) ((a)->load(std::memory_order_relaxed)) +#define qt_atomic_wait_store(a, v) \ + ((a)->store((v), std::memory_order_relaxed +#else #define qt_atomic_wait_t _Atomic uint32_t #define qt_atomic_wait_empty 0u #define qt_atomic_wait_full UINT32_MAX #define qt_atomic_wait_set_empty(a) \ - atomic_store_explicit((a), 0u, memory_order_relaxed) + atomic_store_explicit((a), qt_atomic_wait_empty, memory_order_relaxed) #define qt_atomic_wait_set_full(a) \ - atomic_store_explicit((a), UINT32_MAX, memory_order_relaxed) + atomic_store_explicit((a), qt_atomic_wait_full, memory_order_relaxed) #define qt_atomic_wait_load(a) atomic_load_explicit((a), memory_order_relaxed) #define qt_atomic_wait_store(a, v) \ atomic_store_explicit((a), v, memory_order_relaxed) +#endif // Futex-like atomic wait functionality that's guaranteed to use // the appropriate OS thread pausing functionality (e.g. futex). diff --git a/include/qt_atomics.h b/include/qt_atomics.h index e22d2c490..3cc639b83 100644 --- a/include/qt_atomics.h +++ b/include/qt_atomics.h @@ -4,9 +4,11 @@ #ifndef QT_ATOMICS_H #define QT_ATOMICS_H +// Rely on atomic types getting included from qthread/qthread.h +// It does a little work to navigate vs + #include -#include #include #include @@ -53,9 +55,10 @@ typedef struct qt_spin_exclusive_s { /* added to allow fast critical section ordering */ - aligned_t _Atomic enter; /* and not call pthreads spin_lock -- hard to debug - */ - aligned_t _Atomic exit; /* near the lock under gdb -- 4/1/11 akp */ + QT_Atomic( + aligned_t) enter; /* and not call pthreads spin_lock -- hard to debug + */ + QT_Atomic(aligned_t) exit; /* near the lock under gdb -- 4/1/11 akp */ } qt_spin_exclusive_t; void qt_spin_exclusive_lock(qt_spin_exclusive_t *); @@ -98,7 +101,7 @@ void qt_spin_exclusive_unlock(qt_spin_exclusive_t *); { \ uint32_t val = \ atomic_fetch_add_explicit(&(x)->s.users, 1, memory_order_relaxed); \ - while (val != atomic_load_explicit((_Atomic uint32_t *)&(x)->s.ticket, \ + while (val != atomic_load_explicit((QT_Atomic(uint32_t) *)&(x)->s.ticket, \ memory_order_acquire)) \ SPINLOCK_BODY(); \ } @@ -112,7 +115,7 @@ void qt_spin_exclusive_unlock(qt_spin_exclusive_t *); static inline int QTHREAD_TRYLOCK_TRY(qt_spin_trylock_t *x) { qt_spin_trylock_t newcmp, cmp; uint64_t tmp = - atomic_load_explicit((_Atomic uint64_t *)x, memory_order_relaxed); + atomic_load_explicit((QT_Atomic(uint64_t) *)x, memory_order_relaxed); cmp = *(qt_spin_trylock_t *)&tmp; if (cmp.s.users != cmp.s.ticket) { return 0; } @@ -294,7 +297,7 @@ qthread_internal_incr_mod_(aligned_t *operand, static inline void *qt_internal_atomic_swap_ptr(void **addr, void *newval) { void *oldval = - atomic_load_explicit((void *_Atomic *)addr, memory_order_relaxed); + atomic_load_explicit((QT_Atomic(void *) *)addr, memory_order_relaxed); void *tmp; while ((tmp = qthread_cas_ptr(addr, oldval, newval)) != oldval) { diff --git a/include/qthread/common.h b/include/qthread/common.h index ebfba71e3..f898dc59d 100644 --- a/include/qthread/common.h +++ b/include/qthread/common.h @@ -21,12 +21,14 @@ #endif #ifdef __cplusplus +#ifndef restrict #ifdef __GNUC__ #define restrict __restrict #else #define restrict #endif #endif +#endif #ifdef __GNUC__ #define QTHREAD_TRAP() __builtin_trap() diff --git a/include/qthread/qthread.h b/include/qthread/qthread.h index 5097384d4..a66009e79 100644 --- a/include/qthread/qthread.h +++ b/include/qthread/qthread.h @@ -465,6 +465,19 @@ typedef union qt_spin_trylock_s { QT_Atomic(haligned_t) ticket; QT_Atomic(haligned_t) users; } s; + +#ifdef __cplusplus + // Tell the C++ compiler to not complain about constructors not getting + // called. These things are trivally initializable and mixed-precision + // accesses are allowed on all the architectures we currently care about. + qt_spin_trylock_s() noexcept {} + + qt_spin_trylock_s &operator=(qt_spin_trylock_s const &other) noexcept { + u = other.u; + return *this; + } +#endif + } qt_spin_trylock_t; typedef struct { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ece4b32a7..89d68985f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -83,6 +83,14 @@ set(QTHREADS_SOURCES patterns/wavefront.c ) +set(QTHREADS_BENCHMARK_HELPERS + benchmarks/swapcontext.c +) + +if(QTHREADS_BUILD_BENCHMARKS) + list(APPEND QTHREADS_SOURCES ${QTHREADS_BENCHMARK_HELPERS}) +endif() + add_library(qthread ${QTHREADS_SOURCES}) if ("${QTHREADS_CONTEXT_SWAP_IMPL}" STREQUAL "fastcontext") diff --git a/src/benchmarks/swapcontext.c b/src/benchmarks/swapcontext.c new file mode 100644 index 000000000..7efd6c8bb --- /dev/null +++ b/src/benchmarks/swapcontext.c @@ -0,0 +1,74 @@ +#include +#include + +#include + +#include "qt_context.h" +#include "qt_visibility.h" + +#ifdef USE_SYSTEM_SWAPCONTEXT +#define QT_MAKECONTEXT makecontext +#define QT_GETCONTEXT getcontext +#define QT_SWAPCONTEXT swapcontext +#else +#define QT_MAKECONTEXT qt_makectxt +#define QT_GETCONTEXT getcontext +#define QT_SWAPCONTEXT qt_swapctxt +#endif + +#ifdef QTHREAD_MAKECONTEXT_SPLIT +#error \ + "Context swapping benchmark does not currently support split makecontext." +#endif + +#define SWAP_BENCH_STACK_SIZE 32768u + +typedef struct { + qt_context_t inner; + qt_context_t outer; +} context_pair; + +// This never returns, it just immediately swaps back to +// the outer context (passed as an argument) whenever entered. +static void *ctx_swap_inner(void *arg) { + context_pair *contexts = arg; + while (1) { + // Swap back to the outer one + QT_SWAPCONTEXT(&contexts->inner, &contexts->outer); + } +} + +API_FUNC double qt_ctx_swap_bench(uint64_t num_swaps) { + context_pair contexts; + // Save current context + QT_GETCONTEXT(&contexts.outer); + // Make a context to switch to, running ctx_swap_inner. + // Initialization like this is required by the makecontext API. + // Weird, but okay. + QT_GETCONTEXT(&contexts.inner); + // TODO: can we get away with just using alloca here instead? + contexts.inner.uc_stack.ss_sp = malloc(SWAP_BENCH_STACK_SIZE); + contexts.inner.uc_stack.ss_size = SWAP_BENCH_STACK_SIZE; + QT_MAKECONTEXT( + &contexts.inner, (void (*)(void))&ctx_swap_inner, 1, &contexts); + // Start timer + qtimer_t timer = qtimer_create(); + qtimer_start(timer); + // Actual benchmark. + // Swap into and out of the inner context repeatedly + // without executing any other work. + // This is all on the same thread and everything is small + // enough to at least keep everything in the l1 cace on + // nearly any hardware these days, so this should allow + // us to get a decent estimate of the isolated cost of + // a context swap. + for (uint64_t i = 0u; i < num_swaps / 2; i++) { + // Swap to the inner context; + QT_SWAPCONTEXT(&contexts.outer, &contexts.inner); + } + qtimer_stop(timer); + double time_elapsed = qtimer_secs(timer); + qtimer_destroy(timer); + free(contexts.inner.uc_stack.ss_sp); + return time_elapsed; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1803cf642..89ef303ff 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,10 +1,9 @@ -set(CMAKE_CXX_STANDARD 11) - set(QTHREADS_BUILD_TESTS ON CACHE BOOL "Whether or not to build the qthreads tests.") -if (${QTHREADS_BUILD_TESTS}) - include_directories("." "utils/rng") +if (${QTHREADS_BUILD_TESTS} OR ${QTHREADS_BUILD_BENCHMARKS}) + set(CMAKE_CXX_STANDARD 17) + include_directories("." "utils/rng") add_subdirectory(utils/rng) function(qthreads_test name) @@ -26,7 +25,13 @@ if (${QTHREADS_BUILD_TESTS}) set_property(TEST ${name} PROPERTY ENVIRONMENT "QT_NUM_SHEPHERDS=2;QT_NUM_WORKERS_PER_SHEPHERD=1") set_property(TEST ${name} PROPERTY C_STANDARD "C11") endfunction() +endif() + +if (${QTHREADS_BUILD_BENCHMARKS}) + add_subdirectory(benchmarks) +endif() +if (${QTHREADS_BUILD_TESTS}) add_subdirectory(basics) add_subdirectory(features) add_subdirectory(internal) diff --git a/test/argparsing.h b/test/argparsing.h index 59101bdbd..32db3ed23 100644 --- a/test/argparsing.h +++ b/test/argparsing.h @@ -1,6 +1,14 @@ #ifndef TEST_ARGPARSING_H #define TEST_ARGPARSING_H +#if defined(__cplusplus) && !defined(restrict) +#if defined(__GNUC__) +#define restrict __restrict +#else +#define restrict +#endif +#endif + #if defined(__cplusplus) && __cplusplus < 202302L #include #define ARGP_Atomic(T) std::atomic @@ -123,8 +131,7 @@ inline static void iprintf(char const *restrict format, ...) { va_end(ap); } } - -#endif // if defined(__CYGWIN32__) +#endif #endif // ifndef TEST_ARGPARSING_H /* vim:set expandtab: */ diff --git a/test/benchmarks/CMakeLists.txt b/test/benchmarks/CMakeLists.txt new file mode 100644 index 000000000..8561964fb --- /dev/null +++ b/test/benchmarks/CMakeLists.txt @@ -0,0 +1,19 @@ +# Allow benchmarks to access qthreads internal headers. +# In particular, qt_atomic_wait.h provides platform-generic wrappers around futexes. +include_directories("../../include") + +set(CMAKE_CXX_STANDARD 26) + +qthreads_test_cpp(atomic_accumulate) +qthreads_test_cpp(atomic_exchange_contended) +qthreads_test_cpp(atomic_fetch_min) +qthreads_test_cpp(atomic_fetch_sub) +qthreads_test_cpp(function_call) +qthreads_test_cpp(function_call_unpredicted) +qthreads_test_cpp(nemesis_many_to_one) +qthreads_test_cpp(os_thread_fork_join) +qthreads_test_cpp(os_thread_futex_handoff) +qthreads_test(swapcontext) +qthreads_test(syscall) + +add_subdirectory(generic) diff --git a/test/benchmarks/atomic_accumulate.cpp b/test/benchmarks/atomic_accumulate.cpp new file mode 100644 index 000000000..58df97bdc --- /dev/null +++ b/test/benchmarks/atomic_accumulate.cpp @@ -0,0 +1,20 @@ +#include "atomic_bench.hpp" + +// Similar to atomic reduction operations (See C++ P3111) +std::size_t increment(std::size_t val, + std::atomic &entry) noexcept { + entry.fetch_add(val, std::memory_order_relaxed); + return 1uz; +} + +std::size_t increment_na(std::size_t val, + std::size_t volatile &entry) noexcept { + entry += val; + return 1uz; +} + +int main() { + atomic_bench<0uz, increment>(); + atomic_bench_baseline<0uz, increment_na>(); + return 0; +} diff --git a/test/benchmarks/atomic_bench.hpp b/test/benchmarks/atomic_bench.hpp new file mode 100644 index 000000000..20d4e56d4 --- /dev/null +++ b/test/benchmarks/atomic_bench.hpp @@ -0,0 +1,255 @@ +#ifndef QT_ATOMIC_BENCH_H +#define QT_ATOMIC_BENCH_H + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "argparsing.h" + +// Max page size on currently known systems. +// Used for forcing distinct pages to be allocated +// when doing the non-atomic baseline measurement. +static constexpr std::size_t max_known_page_size = 64uz * 1024uz; + +// Naive implementation to check if two numbers are coprime +// using the Euclidean algorithm. +bool are_coprime(std::size_t a, std::size_t b) noexcept { + if (a < b) return are_coprime(b, a); + std::size_t t; + while (b) { + t = b; + b = a % b; + a = t; + } + return a == 1uz; +} + +// Multiplication by a non-divisor is an easy way to get +// a bijection from the ring of integers mod something to itself. +// This generates "num_requested" bijections on the ring of +// integers mod "base" and stores them in "arr". +// The bijections are not necessarily unique. +void ncoprimes(std::size_t *arr, + std::size_t num_requested, + std::size_t base) noexcept { + if (!base) std::abort(); + if (base < 3uz) { + for (std::size_t i = 0uz; i < num_requested; i++) { arr[i] = 1uz; } + return; + } + std::size_t num_done = 0uz; + std::size_t current = 2uz; + while (num_done < num_requested) { + if (are_coprime(base, current)) arr[num_done++] = current++; + // When simulating high-contention situations, base may be small enough + // that there just aren't enough unique requested coprimes. + // In that case, just let the available coprimes repeate until + // the array is full. + if (current++ == base) current = 2uz; + } +} + +using atomic_reduction_t = std::size_t (*)(std::size_t, + std::atomic_size_t &) noexcept; + +template +void stress_atomics(std::size_t map_id, + std::size_t val, + std::size_t index, + std::atomic *buffer, + std::size_t num_items, + std::size_t num_reps) noexcept { + for (std::size_t i = 0uz; i < num_reps; i++) { + val = red(val, buffer[index]); + index += map_id; + index %= num_items; + } +} + +static std::atomic ready{0z}; +static std::atomic finished{0z}; + +template +void on_thread(std::size_t map_id, + std::size_t val, + std::size_t index, + std::atomic *buffer, + std::size_t num_items, + std::size_t num_reps) noexcept { + ready.fetch_add(1uz, std::memory_order_relaxed); + while (ready.load(std::memory_order_relaxed)) SPINLOCK_BODY(); + stress_atomics(map_id, val, index, buffer, num_items, num_reps); + finished.fetch_add(1uz, memory_order_relaxed); +} + +auto get_args() noexcept { + std::array params{4uz, 100000000uz, 1000000000uz}; + NUMARG(params[0uz], "NUM_THREADS"); + NUMARG(params[1uz], "NUM_ITEMS"); + NUMARG(params[2uz], "NUM_REPS"); + if (!params[1uz] || !params[0uz]) std::abort(); + return params; +} + +template +void atomic_bench() noexcept { + auto [num_threads, num_items, num_reps] = get_args(); + // Check for ill-defined zero cases + std::size_t num_reps_per_thread = num_reps / num_threads; + std::size_t remainder = num_reps % num_threads; + std::size_t map_ids[num_threads]; + ncoprimes(map_ids, num_threads, num_items); + std::size_t max_map_id = *std::max_element(map_ids, map_ids + num_threads); + std::atomic *items = + reinterpret_cast *>( + malloc(sizeof(std::atomic) * num_items)); + for (std::size_t i = 0uz; i < num_items; i++) { + new (&items[i]) std::atomic(init); + } + std::thread pool[num_threads - 1uz]; + // reserve the 0th iteration for the main thread. + std::size_t first_thread_items = + num_reps_per_thread + (remainder ? 1uz : 0uz); + for (std::size_t i = 0uz; i < num_threads - 1uz; i++) { + pool[i] = std::thread(on_thread, + map_ids[i + 1uz], + 0uz, + ((i + 1uz) * num_items) / num_threads, + items, + num_items, + (i + 1uz) < remainder ? num_reps_per_thread + 1uz + : num_reps_per_thread); + } + qtimer_t timer = qtimer_create(); + while (ready.load(std::memory_order_relaxed) < num_threads - 1uz) + SPINLOCK_BODY(); + qtimer_start(timer); + ready.store(0uz, std::memory_order_relaxed); + stress_atomics( + map_ids[0uz], 0uz, 0uz, items, num_items, first_thread_items); + while (finished.load(std::memory_order_relaxed) < num_threads - 1uz) + SPINLOCK_BODY(); + finished.store(0uz, std::memory_order_relaxed); + + qtimer_stop(timer); + double time = qtimer_secs(timer); + printf("Time for running %zu atomic updates to %zu distinct items with %zu " + "threads: %f seconds\n", + num_reps, + num_items, + num_threads, + time); + for (uint64_t i = 0uz; i < num_threads - 1uz; i++) pool[i].join(); + for (uint64_t i = 0uz; i < num_items; i++) items[i].~atomic(); + free(items); +} + +// volatile prevents the compiler from trying to be extra smart and eliminate +// the benchmark loop entirely. +using non_atomic_reduction_t = std::size_t (*)(std::size_t, + std::size_t volatile &); + +template +void stress_non_atomics(std::size_t map_id, + std::size_t val, + std::size_t index, + std::size_t *buffer, + std::size_t num_items, + std::size_t num_reps) noexcept { + for (std::size_t i = 0uz; i < num_reps; i++) { + val = red(val, buffer[index]); + index += map_id; + index %= num_items; + } +} + +template +void on_thread_baseline(std::size_t map_id, + std::size_t val, + std::size_t index, + std::size_t *buffer, + std::size_t num_items, + std::size_t num_reps) noexcept { + // Do initialization of the separate buffers here. + // First-touch NUMA policy will make it so that each thread + // should get pages in its own NUMA domain. + for (std::size_t i = 0uz; i < num_items; i++) buffer[i] = init; + ready.fetch_add(1uz, std::memory_order_relaxed); + while (ready.load(std::memory_order_relaxed)) SPINLOCK_BODY(); + stress_non_atomics(map_id, val, index, buffer, num_items, num_reps); + finished.fetch_sub(1uz, std::memory_order_relaxed); +} + +// A version for doing non-atomic writes to completely disjoint blocks of memory +// in order to measure how much overhead comes from the synchronization. +template +void atomic_bench_baseline() noexcept { + auto [num_threads, num_items, num_reps] = get_args(); + std::size_t num_reps_per_thread = num_reps / num_threads; + std::size_t remainder = num_reps % num_threads; + std::size_t num_items_per_thread = + num_items < num_threads ? 1uz : num_items / num_threads; + // Still replicate the behavior of having different threads jump around at + // different intervals. Just have them do it each within their own distinct + // allocation. + std::size_t map_ids[num_threads]; + ncoprimes(map_ids, num_threads, num_items_per_thread); + std::size_t *items[num_threads]; + // Make sure the allocation is big enough to force each one to have a distinct + // page. + for (std::size_t i = 0uz; i < num_threads; i++) + items[i] = reinterpret_cast(malloc(std::max( + sizeof(std::size_t) * num_items_per_thread, max_known_page_size))); + // Assume first-touch NUMA, so actually do the initialization on the threads + // before running the benchmark. + std::thread pool[num_threads - 1uz]; + // reserve the 0th iteration for the main thread. + std::size_t first_thread_items = + num_reps_per_thread + (remainder ? 1uz : 0uz); + for (std::size_t i = 0uz; i < num_threads - 1uz; i++) { + pool[i] = std::thread(on_thread_baseline, + map_ids[i + 1uz], + 0uz, + ((i + 1uz) * num_items) / num_threads, + items[i], + num_items_per_thread, + (i + 1uz) < remainder ? num_reps_per_thread + 1uz + : num_reps_per_thread); + } + for (std::size_t i = 0uz; i < num_items_per_thread; i++) items[0uz][i] = init; + qtimer_t timer = qtimer_create(); + while (ready.load(std::memory_order_relaxed) < num_threads - 1uz) + SPINLOCK_BODY(); + qtimer_start(timer); + ready.store(0uz, std::memory_order_relaxed); + stress_non_atomics(map_ids[0uz], + 0uz, + 0uz, + items[0uz], + num_items_per_thread, + first_thread_items); + while (finished.load(std::memory_order_relaxed) < num_threads - 1uz) + SPINLOCK_BODY(); + finished.store(0uz, std::memory_order_relaxed); + qtimer_stop(timer); + double time = qtimer_secs(timer); + printf("Baseline time for %zu non-atomic updates to %zu partitioned items " + "with %zu threads: %f seconds\n", + num_reps, + num_items, + num_threads, + time); + for (std::size_t i = 0uz; i < num_threads - 1uz; i++) pool[i].join(); + for (std::size_t i = 0uz; i < num_threads; i++) free(items[i]); +} + +#endif diff --git a/test/benchmarks/atomic_exchange_contended.cpp b/test/benchmarks/atomic_exchange_contended.cpp new file mode 100644 index 000000000..4380eec31 --- /dev/null +++ b/test/benchmarks/atomic_exchange_contended.cpp @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "qt_atomics.h" + +#include "argparsing.h" + +static constexpr std::size_t sentinel = SIZE_MAX; +static std::atomic ready{0z}; +static std::atomic finished{0z}; +static std::atomic tail{sentinel}; + +// A simplification of the communcation pattern that shows up when +// a bunch of threads are pushing work items into a single +// nemesis queue. +void run_swaps(std::atomic *tail, + std::size_t *items, + std::size_t start, + std::size_t num_items) noexcept { + for (std::size_t i = start; i < start + num_items; i++) { + items[i] = tail->exchange(i, std::memory_order_acq_rel); + } +} + +void on_thread(std::atomic *tail, + std::size_t *items, + std::size_t start, + std::size_t num_items) noexcept { + // Indicate that this thread is ready, then wait for the start signal + ready.fetch_add(1uz, std::memory_order_relaxed); + while (ready.load(std::memory_order_relaxed)) SPINLOCK_BODY(); + + // Run the atomic swap loop that's being benchmarked. + run_swaps(tail, items, start, num_items); + + // Indicate that this thread is finished. + finished.fetch_add(1uz, memory_order_relaxed); +} + +int main(int argc, char *argv[]) { + std::size_t num_threads = 4uz, num_items = 1000000000uz; + NUMARG(num_threads, "NUM_THREADS"); + NUMARG(num_items, "NUM_ITEMS"); + std::size_t num_items_per_thread = num_items / num_threads; + std::size_t remainder = num_items % num_threads; + std::size_t *items = + reinterpret_cast(malloc(num_items * sizeof(size_t))); + qtimer_t timer = qtimer_create(); + std::thread pool[num_threads - 1uz]; + // reserve the 0th iteration for the main thread. + std::size_t first_thread_items = + num_items_per_thread + (remainder ? 1uz : 0uz); + std::size_t current_start = first_thread_items; + for (std::size_t i = 0uz; i < num_threads - 1uz; i++) { + std::size_t items_for_this_thread = + num_items_per_thread + (i + 1uz < remainder ? 1uz : 0uz); + pool[i] = std::thread( + on_thread, &tail, items, current_start, items_for_this_thread); + current_start += items_for_this_thread; + } + // Wait for all threads to be ready + printf("waiting for threads\n"); + while (ready.load(std::memory_order_relaxed) < num_threads - 1uz) + SPINLOCK_BODY(); + printf("signaling threads\n"); + // Signal the start to all the threads. + qtimer_start(timer); + ready.store(0uz, std::memory_order_relaxed); + + printf("running main thread work\n"); + // have main thread do the thread 0 work + run_swaps(&tail, items, 0uz, first_thread_items); + + // Wait for all threads to finish + printf("waiting for workers to finish\n"); + while (finished.load(std::memory_order_relaxed) < num_threads - 1uz) + SPINLOCK_BODY(); + + qtimer_stop(timer); + double time = qtimer_secs(timer); + qtimer_destroy(timer); + + for (uint64_t i = 0uz; i < num_threads - 1uz; i++) pool[i].join(); + free(items); + + printf("Time for %lu contended acq_rel swaps by %lu threads: %f seconds.\n", + num_items, + num_threads, + time); +} diff --git a/test/benchmarks/atomic_fetch_min.cpp b/test/benchmarks/atomic_fetch_min.cpp new file mode 100644 index 000000000..8b0cf80c5 --- /dev/null +++ b/test/benchmarks/atomic_fetch_min.cpp @@ -0,0 +1,31 @@ +#include "atomic_bench.hpp" + +// fetch_min is similar to the atomic update done in BFS and SSSP +#ifdef __cpp_lib_atomic_min_max +std::size_t set_earliest(std::size_t val, + std::atomic &entry) noexcept { + entry.fetch_min(val, std::memory_order_relaxed); + return val + 1uz; +} +#else +std::size_t set_earliest(std::size_t val, + std::atomic &entry) noexcept { + std::size_t old_entry = entry.load(std::memory_order_relaxed); + while (old_entry > val && !entry.compare_exchange_weak( + old_entry, val, std::memory_order_relaxed)); + return val + 1uz; +} +#endif + +std::size_t set_earliest_na(std::size_t val, + std::size_t volatile &entry) noexcept { + if (val < entry) entry = val; + return val + 1uz; +} + +int main() { + atomic_bench::max(), set_earliest>(); + atomic_bench_baseline::max(), + set_earliest_na>(); + return 0; +} diff --git a/test/benchmarks/atomic_fetch_sub.cpp b/test/benchmarks/atomic_fetch_sub.cpp new file mode 100644 index 000000000..935cf104a --- /dev/null +++ b/test/benchmarks/atomic_fetch_sub.cpp @@ -0,0 +1,26 @@ +#include "atomic_bench.hpp" + +// Similar to the dependency computation idiom in fine-grained DAG execution. +// Also similar to atomic reference counting synchronization. +std::size_t decrement_and_reset(std::size_t val, + std::atomic &entry) noexcept { + if (!entry.fetch_sub(val, std::memory_order_relaxed)) entry.fetch_add(5uz); + return 1uz; +} + +std::size_t decrement_and_reset_na(std::size_t val, + std::size_t volatile &entry) noexcept { + // Have to cast back to non-volatile to make the compiler happy. + // It's volatile in the signature just to force a cold read from memory. + // In this case, we have to use a fence instead. + __asm__ __volatile__("" ::: "memory"); + std::size_t &entry_nv = const_cast(entry); + if (!(entry_nv -= val)) entry = 5uz; + return 1uz; +} + +int main() { + atomic_bench<5uz, decrement_and_reset>(); + atomic_bench_baseline<5uz, decrement_and_reset_na>(); + return 0; +} diff --git a/test/benchmarks/function_call.cpp b/test/benchmarks/function_call.cpp new file mode 100644 index 000000000..a92fa06be --- /dev/null +++ b/test/benchmarks/function_call.cpp @@ -0,0 +1,101 @@ +#include +#include +#include +#include + +#include + +#include "argparsing.h" + +// Only for functions. +// Call a function but force it not to be inlined. +template +auto invoke_noinline(F *f, + As... as) noexcept(noexcept(f(std::forward(as)...))) { + auto vf = reinterpret_cast(f); + return vf(std::forward(as)...); +} + +template +struct nargs_empty; + +template +struct nargs_empty> { + // Use inside a pack expansion to just repeat t for each entry in the pack. + template + using repeat_for_pack = t; + + // Empty function with + static void empty(repeat_for_pack...) noexcept {} + + // Return first argument. + // Again, for use in a pack expansion for repeating a value. + template + static auto first(T a, T b) noexcept { + return a; + } + + static void call_empty(std::size_t j) noexcept { + // Force the call through a volatile pointer to prevent inlining. + invoke_noinline(&empty, first(j, i)...); + } +}; + +template +void bench(std::size_t num_reps, qtimer_t timer) noexcept { + qtimer_start(timer); + for (uint64_t i = 0u; i < num_reps; i++) { + nargs_empty>::call_empty(i); + } + qtimer_stop(timer); + double time = qtimer_secs(timer); + printf( + "Time for %lu function calls with %lu uint64_t arguments is: %f seconds\n", + num_reps, + nargs, + time); +} + +template +void bench_at_sizes(std::size_t num_reps, qtimer_t timer) noexcept { + (bench(num_reps, timer), ...); +} + +template +void bench_at_sizes_from_sequence(std::size_t num_reps, qtimer_t timer); + +template +void bench_at_sizes_from_sequence(std::index_sequence, + std::size_t num_reps, + qtimer_t timer) noexcept { + bench_at_sizes(num_reps, timer); +} + +template +struct index_range_impl; + +template +struct index_range_impl> { + using impl = std::index_sequence<(start + step * entries)...>; +}; + +template +using index_range = typename index_range_impl< + start, + step, + std::make_index_sequence<(start <= stop ? (stop - start) / step : 0)>>::impl; + +template +void bench_at_size_range(std::size_t num_reps, qtimer_t timer) noexcept { + bench_at_sizes_from_sequence( + index_range(), num_reps, timer); +} + +int main(int argc, char *argv[]) { + uint64_t num_reps = 1000000ull; + NUMARG(num_reps, "NUM_REPS"); + qtimer_t timer = qtimer_create(); + bench_at_size_range<2, 66, 2>(num_reps, timer); + + qtimer_destroy(timer); +} diff --git a/test/benchmarks/function_call_unpredicted.cpp b/test/benchmarks/function_call_unpredicted.cpp new file mode 100644 index 000000000..89e157de0 --- /dev/null +++ b/test/benchmarks/function_call_unpredicted.cpp @@ -0,0 +1,86 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "argparsing.h" + +// Design: at each iteration call into an entry in an array +// of volatile function pointers. The volatile prevents +// inlining. Get the next index to use based on some cheap +// computations inside each function called. +// The hope is that that it's enough to confuse the branch predictor. +// Get the next index to use based on the current index of +// iteration and the previous +// The compiler's strength reduction pass should make the +// mod operation very cheap, though it's probably a good idea +// to only mod by powers of two just to be safe. + +template +uint64_t add_n(std::size_t i, std::size_t current) noexcept { + return (current + i + offset) % mod; +} + +using fptr_t = decltype(&add_n<0, 2>); + +template +auto gen_array_impl(std::index_sequence) noexcept { + return std::array{ + add_n...}; +} + +template +auto gen_array() noexcept { + return gen_array_impl(std::make_index_sequence()); +} + +void baseline(std::size_t num_reps, qtimer_t timer) noexcept { + std::size_t current = 0; + fptr_t volatile fptr = &add_n<1, 4>; + qtimer_start(timer); + for (std::size_t i = 0; i < num_reps; i++) { current = fptr(i, current); } + qtimer_stop(timer); + double time = qtimer_secs(timer); + printf( + "Baseline time for %zu function calls is: %f seconds.\n", num_reps, time); +} + +template +void bench(std::size_t num_reps, qtimer_t timer) noexcept { + static_assert(fptr_array_size > 0ull, + "Need nonzero array size. There has to be something to call."); + std::size_t current = 0; + auto fptr_array = gen_array(); + qtimer_start(timer); + for (std::size_t i = 0; i < num_reps; i++) { + current = fptr_array[current](i, current); + } + qtimer_stop(timer); + double time = qtimer_secs(timer); + printf("Time for %zu function calls (with array size %zu) without branch " + "predicts is: %f seconds.\n", + num_reps, + fptr_array_size, + time); +} + +template +void bench_at_sizes(std::size_t num_reps, qtimer_t timer) noexcept { + (bench(num_reps, timer), ...); +} + +int main(int argc, char *argv[]) { + std::size_t num_reps = 1000000ull; + NUMARG(num_reps, "NUM_REPS"); + qtimer_t timer = qtimer_create(); + + baseline(num_reps, timer); + bench_at_sizes<2, 4, 8, 16, 32, 64, 128, 256, 512, 1024>(num_reps, timer); + + qtimer_destroy(timer); +} diff --git a/test/benchmarks/generic/time_thread_ring.c b/test/benchmarks/generic/time_thread_ring.c index a003d49df..24d87f3e6 100644 --- a/test/benchmarks/generic/time_thread_ring.c +++ b/test/benchmarks/generic/time_thread_ring.c @@ -8,26 +8,24 @@ // native qthreads version of thread-ring, based on Chapel's release version // https://github.com/chapel-lang/chapel/blob/master/test/release/examples/benchmarks/shootout/threadring.chpl -// static int n = 1000, ntasks = 503; -static int n = 50000000, ntasks = 503; +static int n = 1000, ntasks = 503; +// static int n = 50000000, ntasks = 503; static aligned_t *mailbox; static void passTokens(size_t start, size_t stop, void *arg) { uint64_t id = start; - // printf("entering tokens\n"); uint64_t numPasses = 0; do { - // printf("numpasses %lld n %d\n", numPasses, n); qthread_readFE(&numPasses, &mailbox[id]); qthread_writeEF_const(&mailbox[(id + 1) % ntasks], numPasses + 1); if (numPasses == n) { printf("%ld\n", (long)id + 1); } } while (numPasses < n); - // printf("exiting tokens\n"); } static void init() { - assert(qthread_initialize() == 0); + int status = qthread_initialize(); + if (status) abort(); printf("%i threads...\n", qthread_num_workers()); // init array of syncvars (code grabbed from stress/feb_stream.c) @@ -49,9 +47,8 @@ int main(int argc, char **argv) { qtimer_stop(timer); ring_time = qtimer_secs(timer); - printf("\tThread ring time: %f usecs, %f/sec\n", - 1000000 * ring_time / ntasks, - ntasks / ring_time); + printf( + "\tThread ring time: %f secs, %f/sec\n", ring_time, ntasks / ring_time); return 0; } diff --git a/test/benchmarks/nemesis_many_to_one.cpp b/test/benchmarks/nemesis_many_to_one.cpp new file mode 100644 index 000000000..bfea391ea --- /dev/null +++ b/test/benchmarks/nemesis_many_to_one.cpp @@ -0,0 +1,85 @@ +#include +#include + +#include +#include + +#include + +#include "argparsing.h" + +#include "nemesis_mini.hpp" + +using queue_t = nemesis_queue; +using item_t = queue_t::item_t; + +void producer_thread(std::atomic *ready, + queue_t *queue, + std::size_t items_per_thread, + item_t *item_buffer) noexcept { + while (!ready->load(std::memory_order_relaxed)) SPINLOCK_BODY(); + for (std::size_t i = 0ull; i < items_per_thread; i++) { + item_t *item = new (&item_buffer[i]) item_t{i + 1ull}; + queue->enqueue_existing(item); + } +} + +std::size_t arithmetic_sum(std::size_t val) { + if (val % 2ull) { + return ((val - 1ull) / 2ull + 1ull) * val; + } else { + return (val / 2ull) * (val + 1ull); + } +} + +int main() { + std::size_t num_items = 1000000ull; + std::size_t num_threads = 4u; + NUMARG(num_items, "NUM_ITEMS"); + NUMARG(num_threads, "NUM_THREADS"); + std::size_t num_items_per_thread = num_items / num_threads; + std::size_t remainder = num_items % num_threads; + std::atomic ready{false}; + std::thread pool[num_threads]; + // Defer proper initialization to inside each worker thread. + item_t *items = (item_t *)malloc(sizeof(item_t) * num_items); + queue_t queue; + qtimer_t timer = qtimer_create(); + qtimer_start(timer); + std::size_t current = 0ull; + for (std::size_t i = 0ull; i < num_threads; i++) { + std::size_t items_for_this_thread = + num_items_per_thread + std::size_t(i < remainder); + pool[i] = std::thread( + producer_thread, &ready, &queue, items_for_this_thread, items + current); + current += items_for_this_thread; + } + ready.store(true, std::memory_order_relaxed); + qtimer_start(timer); + std::size_t i = 0ull; + std::size_t sum = 0ull; + while (i < num_items) { + std::size_t fail_count = 0ull; + item_t *item = queue.dequeue_single(); + if (item) { + sum += item->value; + i++; + } else { + fail_count++; + if (fail_count >= 10000) { printf("failed at index: %lu\n", i); } + } + } + qtimer_stop(timer); + double time = qtimer_secs(timer); + printf("Time for running %lu work items from %lu distinct threads through a " + "nemesis queue to a single consumer is: %f seconds\n", + num_items, + num_threads, + time); + for (std::size_t i = 0ull; i < num_threads; i++) { pool[i].join(); } + free(items); + std::size_t expected = arithmetic_sum(num_items_per_thread) * (num_threads) + + remainder * (num_items_per_thread + 1uz); + if (expected != sum) std::abort(); +} + diff --git a/test/benchmarks/nemesis_mini.hpp b/test/benchmarks/nemesis_mini.hpp new file mode 100644 index 000000000..5840a857c --- /dev/null +++ b/test/benchmarks/nemesis_mini.hpp @@ -0,0 +1,87 @@ +#ifndef QT_MINI_NEMESIS_H +#define QT_MINI_NEMESIS_H + +// A miniaturized version of the nemesis threadqueue usable for benchmarking. + +#include +#include +#include +#include + +#define CACHE_LINE_SIZE 128 + +// TODO: for platforms with 128b atomics and mixed-size coherency +// (most of them as of this writing), test whether some of these +// operations can/should be shited to the 128-bit instructions. +// It likely doesn't matter since the synchronization +// is probably the main overhead and it's probably done at a +// more coarse level than 8-byte blocks, but it's good to test. + +template +struct nemesis_queue { + static_assert(std::is_same_v>); + static_assert(std::is_trivially_destructible_v); + static_assert(std::is_trivially_destructible_v>); + + struct item_t { + std::atomic next; + std::atomic value; + item_t(item_t const &) = delete; + item_t(item_t &&) = delete; + + item_t(value_t val) noexcept: next{nullptr}, value{val} {} + }; + + std::atomic head; +#ifdef HEAD_TAIL_PADDING + char scratch_head_to_tail[CACHE_LINE_SIZE - sizeof(head)]; +#endif + std::atomic tail; + + nemesis_queue() noexcept: head(nullptr), tail(nullptr) { + head.store(nullptr, std::memory_order_relaxed); + tail.store(nullptr, std::memory_order_relaxed); + } + + void enqueue_existing(item_t *item) noexcept { + assert(!item->next.load(memory_order_relaxed) && + "Encountered null next pointer"); + // Acquire/release ordering needed here because the item + // may be recently initialized and we need those values to + // be committed to memory to ensure correct ordering of writes + // if another thread is going to immediately come and set + // the next pointer as well. + item_t *old_tail = tail.exchange(item, std::memory_order_acq_rel); + if (old_tail) { + old_tail->next.store(item, std::memory_order_relaxed); + } else { + head.store(item, std::memory_order_relaxed); + } + } + + item_t *dequeue_single() noexcept { + // Acquire/release ordering here prevents picking up + // stale values in the item (e.g. from initializaton). + // This pairs with the fence on the main exchange operation. + item_t *item = head.load(std::memory_order_acquire); + if (!item) return nullptr; + item_t *next = item->next.load(std::memory_order_relaxed); + if (next) { + head.store(next, std::memory_order_relaxed); + } else { + head.store(nullptr, std::memory_order_relaxed); + item_t *tail_local = item; + if (!tail.compare_exchange_strong(tail_local, + nullptr, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + next = item->next.load(std::memory_order_relaxed); + while (!next) next = item->next.load(std::memory_order_relaxed); + head.store(next, std::memory_order_relaxed); + } + } + return item; + } +}; + +#endif // QT_MINI_NEMESIS_H diff --git a/test/benchmarks/os_thread_fork_join.cpp b/test/benchmarks/os_thread_fork_join.cpp new file mode 100644 index 000000000..124f17d34 --- /dev/null +++ b/test/benchmarks/os_thread_fork_join.cpp @@ -0,0 +1,26 @@ +#include +#include +#include +#include + +#include + +#include "argparsing.h" + +int main(int argc, char *argv[]) { + uint64_t num_reps = 1000ull; + NUMARG(num_reps, "NUM_REPS"); + + qtimer_t timer = qtimer_create(); + qtimer_start(timer); + for (uint64_t i = 0u; i < num_reps; i++) { + std::thread t{[]() noexcept {}}; + t.join(); + } + qtimer_stop(timer); + double time = qtimer_secs(timer); + qtimer_destroy(timer); + + printf( + "Time for %lu thread launch/join cycles is: %f seconds.\n", num_reps, time); +} diff --git a/test/benchmarks/os_thread_futex_handoff.cpp b/test/benchmarks/os_thread_futex_handoff.cpp new file mode 100644 index 000000000..9b2866221 --- /dev/null +++ b/test/benchmarks/os_thread_futex_handoff.cpp @@ -0,0 +1,46 @@ +#include +#include +#include +#include + +#include + +#include "qt_atomic_wait.h" + +#include "argparsing.h" + +qt_atomic_wait_t flag{qt_atomic_wait_empty}; + +void thread_func(qt_atomic_wait_t *flag_ptr, uint64_t num_reps) noexcept { + for (uint64_t i = 0u; i < num_reps / 2u; i++) { + do { + qt_wait_on_address(flag_ptr, qt_atomic_wait_empty); + } while (qt_atomic_wait_load(flag_ptr) == qt_atomic_wait_empty); + qt_atomic_wait_set_empty(flag_ptr); + qt_wake_all(flag_ptr); + } +} + +int main(int argc, char *argv[]) { + uint64_t num_reps = 1000ull; + NUMARG(num_reps, "NUM_REPS"); + + std::thread t{thread_func, &flag, num_reps}; + + qtimer_t timer = qtimer_create(); + qtimer_start(timer); + for (uint64_t i = 0u; i < num_reps / 2u; i++) { + qt_atomic_wait_set_full(&flag); + qt_wake_all(&flag); + do { + qt_wait_on_address(&flag, qt_atomic_wait_full); + } while (qt_atomic_wait_load(&flag) == qt_atomic_wait_full); + } + qtimer_stop(timer); + double time = qtimer_secs(timer); + qtimer_destroy(timer); + + t.join(); + printf( + "Time for %lu thread futex wait cycles is: %f seconds.\n", num_reps, time); +} diff --git a/test/benchmarks/swapcontext.c b/test/benchmarks/swapcontext.c new file mode 100644 index 000000000..d410eee2a --- /dev/null +++ b/test/benchmarks/swapcontext.c @@ -0,0 +1,17 @@ +#include +#include + +#include "argparsing.h" + +extern double qt_ctx_swap_bench(uint64_t num_swaps); + +int main(int argc, char *argv[]) { + uint64_t num_swaps = 1000000ull; + NUMARG(num_swaps, "NUM_SWAPS"); + if (num_swaps % 2ull) { + iprintf("Error: number of context swaps must be even.\n"); + abort(); + } + double time = qt_ctx_swap_bench(num_swaps); + printf("Time for %lu context swaps is: %f seconds.\n", num_swaps, time); +} diff --git a/test/benchmarks/syscall.c b/test/benchmarks/syscall.c new file mode 100644 index 000000000..e6ed7987e --- /dev/null +++ b/test/benchmarks/syscall.c @@ -0,0 +1,22 @@ +#include +#include + +#include + +#include + +#include "argparsing.h" + +int main(int argc, char *argv[]) { + uint64_t num_reps = 1000000ull; + NUMARG(num_reps, "NUM_REPS"); + + qtimer_t timer = qtimer_create(); + qtimer_start(timer); + for (uint64_t i = 0u; i < num_reps; i++) { getpid(); } + qtimer_stop(timer); + double time = qtimer_secs(timer); + qtimer_destroy(timer); + + printf("Time for %lu syscalls is: %f seconds.\n", num_reps, time); +}