diff --git a/lib/cgroup_bw.bpf.c b/lib/cgroup_bw.bpf.c index eea27baf34..19e33293e8 100644 --- a/lib/cgroup_bw.bpf.c +++ b/lib/cgroup_bw.bpf.c @@ -14,6 +14,25 @@ #define U64_MAX ((u64)~0ULL) #endif +/* + * Kernel signal/jobctl flag bits. Mirror of include/linux/sched/signal.h + * and include/linux/sched/jobctl.h; vmlinux.h only carries types, not CPP + * macros, so we redeclare the bits we read here. These are stable ABI in + * practice — the kernel exposes them to userspace via task state and ptrace + * interfaces. + */ +#ifndef SIGNAL_GROUP_EXIT +#define SIGNAL_GROUP_EXIT 0x00000004 +#endif +#ifndef JOBCTL_STOP_PENDING +#define JOBCTL_STOP_PENDING (1UL << 17) +#define JOBCTL_TRAP_STOP (1UL << 19) +#define JOBCTL_TRAP_NOTIFY (1UL << 20) +#define JOBCTL_TRAP_FREEZE (1UL << 23) +#define JOBCTL_TRAP_MASK (JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY) +#define JOBCTL_PENDING_MASK (JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK) +#endif + extern int scx_cgroup_bw_enqueue_cb(u64 taskc); enum scx_cgroup_consts { @@ -47,18 +66,51 @@ enum scx_cgroup_consts { CBW_NR_CGRP_MAX = 2048, /* maximum number of scx_cgroup_llc_ctx: 2048 cgroups * 32 LLCs */ CBW_NR_CGRP_LLC_MAX = (CBW_NR_CGRP_MAX * 32), - /* The maximum height of a cgroup tree. - * cgroupv2 default maximum depth is 32 (kernel CGROUPS_DEPTH_MAX). */ - CBW_CGRP_TREE_HEIGHT_MAX = 32, + /* The maximum height of a cgroup tree. */ + CBW_CGRP_TREE_HEIGHT_MAX = 64, /* unlimited quota ("max") from scx_cgroup_init_args and scx_cgroup_bw_set() */ CBW_RUNTUME_INF_RAW = ((u64)~0ULL), /* unlimited quota ("max"); This is for easier comparison between signed vs. unsigned integers. */ CBW_RUNTUME_INF = ((s64)~((u64)1 << 63)), /* maximum number of re-enqueue tasks in one dispatch */ CBW_REENQ_MAX_BATCH = 2, - /* size of the deferred BTQ destroy queue */ - CBW_DEFERRED_BTQ_SIZE = 256, + /* + * Size of the deferred BTQ destroy queue. + * CBW_NR_CGRP_LLC_MAX * 2 = 131072 slots, 1024 KB. + */ + CBW_DEFERRED_BTQ_SIZE = (CBW_NR_CGRP_LLC_MAX * 2), + /* + * Scheduling pressure hint, 1024-scale. The BPF scheduler scales a + * task's time slice as: + * + * slice = (base_slice * CBW_PRESSURE_SCALE) / pressure + * + * CBW_PRESSURE_NORMAL (== CBW_PRESSURE_SCALE) means no reduction. + * CBW_PRESSURE_MAX caps the reduction at 1/16 of the base slice. + * + * Two signals combine via max(): + * Budget pressure: hyperbolic below CBW_PRESSURE_BUDGET_KNEE (25%). + * Backlog pressure: linear with BTQ depth, capped at + * CBW_PRESSURE_BACKLOG_CAP tasks. + * 1024 + 128 * 128 = 17408 → clamped to MAX. + */ + CBW_PRESSURE_SCALE = 1024, + CBW_PRESSURE_NORMAL = CBW_PRESSURE_SCALE, + CBW_PRESSURE_MAX = 16384, + CBW_PRESSURE_BUDGET_KNEE = 256, + CBW_PRESSURE_BACKLOG_STEP = 128, + CBW_PRESSURE_BACKLOG_CAP = 128, + /* + * Shift used to split the 64-bit BTQ key evenly into a wall-clock + * epoch (upper 32 bits) and a scheduler vtime (lower 32 bits). + * Tasks queued in the same ~4-second epoch (2^32 ns ~= 4.29 s) + * compete by their vtime; tasks from an earlier epoch are always + * dispatched first, bounding the maximum BTQ wait to ~4 seconds. + */ + CBW_BTQ_VTIME_MASK_SHIFT = 32, }; +#define CBW_BTQ_VTIME_LOWER_MASK ((1ULL << CBW_BTQ_VTIME_MASK_SHIFT) - 1ULL) +#define CBW_BTQ_VTIME_UPPER_MASK ((u64)~CBW_BTQ_VTIME_LOWER_MASK) /* * Root cgroup id. This is the kernel-level cgroup_id of the @@ -126,6 +178,14 @@ struct scx_cgroup_ctx { * treated as read-only in the hot path. */ bool has_llcx; + + /* + * Scheduling pressure hint (CBW_PRESSURE_SCALE = 1024 means no + * reduction). Written at ~10 Hz by cbw_replenish_cgroup(); fits + * in the 7-byte natural padding after has_llcx so this cache + * line stays at 64 bytes. + */ + u32 pressure; } __attribute__((aligned(SCX_CACHELINE_SIZE))); /* read-write cache line */ @@ -202,6 +262,16 @@ struct scx_cgroup_ctx { * meaningful consumption rate to track. */ u64 avg_consumption_rate; + + /* + * Total number of tasks queued in the BTQs across all LLC + * domains, sampled at the last replenishment boundary by + * cbw_replenish_cgroup(). Feeds the backlog component of + * cbw_compute_pressure(); also exported through + * cbw_dump_cgroup() so operators can see the per-cgroup + * backlog alongside the throttle counters. + */ + u32 nr_pending; } __attribute__((aligned(SCX_CACHELINE_SIZE))); } __attribute__((aligned(SCX_CACHELINE_SIZE))); @@ -809,6 +879,71 @@ scx_cgroup_llc_ctx_t *cbw_get_llc_ctx(struct cgroup *cgrp, int llc_id) return cbw_get_llc_ctx_with_id(cgroup_get_id(cgrp), llc_id); } +static __always_inline +u64 cbw_taskc_get_cgx_raw(scx_task_cgroup_bw_t *taskc, u64 cgrp_id) +{ + u64 cgx_raw; + + /* + * On cache hit, return taskc->cgx_raw directly. On miss, look up + * via cbw_get_cgroup_ctx_raw() and populate taskc->cgx_raw if + * @taskc is non-NULL. A miss means the CPU controller is disabled + * for this cgroup, or the cgroup is mid-teardown -- the caller + * chooses how to react (-ESRCH vs silent skip). + */ + if (taskc && taskc->cgx_raw) + return taskc->cgx_raw; + + cgx_raw = cbw_get_cgroup_ctx_raw(cgrp_id); + if (!cgx_raw) + return 0; + + if (taskc) + taskc->cgx_raw = cgx_raw; + + return cgx_raw; +} + +static __always_inline +u64 cbw_taskc_get_llcx_raw(scx_task_cgroup_bw_t *taskc, u64 cgrp_id, int llc_id) +{ + u64 llcx_raw; + + /* + * Cache key is (cgrp_id, llc_id); cgrp_id is implicit because + * @taskc is invalidated on cgroup migration (scx_cgroup_bw_move), + * so a non-zero llcx_raw always belongs to the current cgroup. + * On LLC change, re-look-up via cbw_get_llc_ctx_raw_with_id() and + * refresh both fields. + */ + if (taskc->llcx_raw && taskc->last_llc_id == llc_id) + return taskc->llcx_raw; + + llcx_raw = cbw_get_llc_ctx_raw_with_id(cgrp_id, llc_id); + if (!llcx_raw) + return 0; + + taskc->llcx_raw = llcx_raw; + taskc->last_llc_id = llc_id; + + return llcx_raw; +} + +static __always_inline +void cbw_taskc_invalidate(scx_task_cgroup_bw_t *taskc) +{ + /* + * Called on cgroup migration (scx_cgroup_bw_move) and on cgroup + * teardown scans. Use __sync_lock_test_and_set instead of plain + * stores: LLVM may fold constant stores into base+offset addressing + * and omit addr_space_cast for the arena pointer, which the BPF + * verifier rejects. Atomics always emit addr_space_cast for the + * base register regardless of offset. + */ + __sync_lock_test_and_set(&taskc->cgx_raw, 0); + __sync_lock_test_and_set(&taskc->llcx_raw, 0); +} + static long cbw_del_llc_ctx_with_id(u64 cgrp_id, int llc_id) { @@ -957,8 +1092,7 @@ int cbw_free_llc_ctx(scx_cgroup_ctx_t *cgx, u64 cgrp_id) * path's lock-acquire provides the matching * load-acquire. */ - WRITE_ONCE(t->cgx_raw, 0); - WRITE_ONCE(t->llcx_raw, 0); + cbw_taskc_invalidate(t); /* * Set task's vtime to zero so we can reap the * the throttled exiting task as soon as possible. @@ -1098,10 +1232,12 @@ int scx_cgroup_bw_init(struct cgroup *cgrp __arg_trusted, struct scx_cgroup_init struct cgroup *parent; u64 cgrp_id; - cbw_dbg_cgrp(" level: %d -- period_us: %llu -- quota_us: %llu -- burst_us: %llu ", - cgrp->level, args->bw_period_us, args->bw_quota_us, args->bw_burst_us); - cgrp_id = cgroup_get_id(cgrp); + if (unlikely(cgrp->level >= CBW_CGRP_TREE_HEIGHT_MAX)) { + cbw_err("cgroup is too deep: cgid: %llu, level: %d", + cgrp_id, cgrp->level); + return -ENOMEM; + } /* * Allocate and initialize scx_cgroup_ctx for @cgrp. @@ -1130,6 +1266,8 @@ int scx_cgroup_bw_init(struct cgroup *cgrp __arg_trusted, struct scx_cgroup_init cgx->runtime_total_sloppy = 0; cgx->period_budget = cgx->nquota_ub; cgx->is_throttled = false; + cgx->pressure = CBW_PRESSURE_NORMAL; + cgx->nr_pending = 0; /* * The parent of @cgrp becomes non-leaf. If the parent is not @@ -1226,7 +1364,6 @@ int cbw_cgroup_bw_offline(u64 cgrp_id) * scx_cgroup_bw_reenqueue(), both of which already have deep call * chains. */ - cbw_dbg("Offline a cgroup: %llu", cgrp_id); cbw_unthrottle_cgroup_for_exit(cgrp_id); return cbw_free_llc_ctx(NULL, cgrp_id); } @@ -1246,8 +1383,6 @@ int scx_cgroup_bw_exit(struct cgroup *cgrp __arg_trusted) int ret = 0; u64 cgrp_id; - cbw_dbg_cgrp(); - /* * A cgroup can exit when there are exiting tasks (TASK_DEAD) under it, * because the kernel does not count them as living tasks. So, care @@ -1292,8 +1427,6 @@ int scx_cgroup_bw_set(struct cgroup *cgrp __arg_trusted, u64 period_us, u64 quot struct cgroup_subsys_state *start_css, *pos; int ret = 0; - cbw_dbg_cgrp(); - /* Update the cgroup's bandwidth. */ cgx_raw = cbw_get_cgroup_ctx_raw(cgroup_get_id(cgrp)); if (!cgx_raw) { @@ -1499,9 +1632,6 @@ int cbw_update_runtime_total_sloppy(struct cgroup *cgrp) /* Update the previous level. */ prev_level = cur_level; - - cbw_dbg("cgid%llu -- rt_llcx: %lld -- runtime_total_sloppy: %lld", - cur_cgx->id, rt_llcx, cur_cgx->runtime_total_sloppy); } bpf_rcu_read_unlock(); @@ -1661,24 +1791,16 @@ int cbw_cgroup_bw_throttled(u64 cgrp_id, u64 taskc_raw) if (unlikely(cgrp_id == 0)) return 0; - if (taskc && taskc->cgx_raw) { - cgx_raw = taskc->cgx_raw; - } else { - cgx_raw = cbw_get_cgroup_ctx_raw(cgrp_id); - if (!cgx_raw) { - /* - * The CPU controller is not enabled for this cgroup. - */ - cbw_dbg("Failed to lookup a cgroup ctx: %llu", cgrp_id); - return -ESRCH; - } - if (taskc) - taskc->cgx_raw = cgx_raw; + cgx_raw = cbw_taskc_get_cgx_raw(taskc, cgrp_id); + if (!cgx_raw) { + /* + * The CPU controller is not enabled for this cgroup. + */ + return -ESRCH; } cgx = (scx_cgroup_ctx_t *)cgx_raw; if (READ_ONCE(cgx->is_throttled)) { - dbg_cgx(cgx, "throttled: "); return -EAGAIN; } @@ -1701,16 +1823,76 @@ int scx_cgroup_bw_throttled(u64 cgrp_id, struct task_struct *p __arg_trusted, u64 taskc) { /* - * Never throttle an exiting task. In do_exit(), a task is removed from - * the PID map by __unhash_process() (called from exit_notify()) in the - * window between PF_EXITING being set and TASK_DEAD being set. If the - * task is preempted in this window and throttled into the BTQ, the BTQ - * drain calls scx_cgroup_bw_enqueue_cb() to reenqueue it. The callback - * looks up the task pointer via bpf_task_from_pid(), which returns NULL - * for an unhashed task. With no way to reenqueue it, the task is - * permanently lost from all runqueues, causing a watchdog timeout. + * Bypass throttling for tasks the kernel is about to take out of + * SCX, or that will run only briefly to complete a kernel-mediated + * transition. Parking such a task in the BTQ either loses it + * (PF_EXITING / SIGNAL_GROUP_EXIT — kernel unhashes the task before + * drain can find it) or makes a kernel-visible operation (signal + * stop, ptrace trap, cgroup-v2 freeze) appear to stall. Quota + * impact is negligible: these tasks consume essentially no CPU + * before leaving SCX. + */ + + /* + * PF_EXITING: task is in do_exit(), being unhashed from the PID + * hash and torn down. __unhash_process() runs between PF_EXITING + * being set and TASK_DEAD being set; if we park the task in the BTQ + * during that window, the drain's bpf_task_from_pid() returns NULL, + * the task is lost from all runqueues, and the watchdog stalls. + */ + if (unlikely(p->flags & PF_EXITING)) + return 0; + + /* + * SIGNAL_GROUP_EXIT: the thread group is exiting (SIGKILL or + * exit_group()) and is propagating the exit to every thread. + * Narrow window where the group flag is set but PF_EXITING has not + * yet landed on a given sibling; throttling here has the same + * "lost task" risk as PF_EXITING. */ - if (p->flags & PF_EXITING) + if (unlikely(READ_ONCE(p->signal->flags) & SIGNAL_GROUP_EXIT)) + return 0; + + /* + * Jobctl bits — kernel says "this task is about to make a brief + * kernel-mediated transition; let it run long enough to observe + * and act on the pending event." Throttling here doesn't lose the + * task, but delays the user-visible effect of the operation. + * + * JOBCTL_PENDING_MASK = JOBCTL_STOP_PENDING | JOBCTL_TRAP_MASK + * JOBCTL_TRAP_MASK = JOBCTL_TRAP_STOP | JOBCTL_TRAP_NOTIFY + * + * JOBCTL_STOP_PENDING: group SIGSTOP is pending — siblings have + * been signal_wake_up()'d so they can + * observe the stop and transition to + * TASK_STOPPED. Throttling delays SIGSTOP + * delivery by the throttle window. + * + * JOBCTL_TRAP_STOP: ptrace stop trap pending — debugger is + * waiting for the tracee to stop. + * Throttling makes the debugger appear hung. + * + * JOBCTL_TRAP_NOTIFY: ptrace notify trap pending (e.g. seccomp, + * PTRACE_EVENT_*); same shape as TRAP_STOP. + * + * JOBCTL_TRAP_FREEZE: cgroup-v2 freezer is engaging and every + * thread under the cgroup must trap into + * the freezer. Throttling delays + * convergence — `cgroup.freeze` write + * appears to hang. This bit is outside + * JOBCTL_PENDING_MASK, so it needs its own + * test alongside the mask. + * + * Using the mask (rather than spelling out STOP/NOTIFY individually) + * keeps this aligned with the kernel's own grouping; new bits added + * to JOBCTL_PENDING_MASK get picked up automatically. + * + * READ_ONCE: p->jobctl and p->signal->flags are written under + * siglock on a different CPU; without the barrier the BPF compiler + * may reorder or fold the load. + */ + if (unlikely(READ_ONCE(p->jobctl) & + (JOBCTL_PENDING_MASK | JOBCTL_TRAP_FREEZE))) return 0; return cbw_cgroup_bw_throttled(cgrp_id, taskc); @@ -1756,17 +1938,9 @@ int scx_cgroup_bw_consume(u64 cgrp_id, u64 consumed_ns, u64 taskc_raw) goto accounting_out; } - /* - * Ensure cgx_raw is cached; populate it on the first call. - */ - if (taskc->cgx_raw) { - cgx_raw = taskc->cgx_raw; - } else { - cgx_raw = cbw_get_cgroup_ctx_raw(cgrp_id); - if (!cgx_raw) - return 0; - taskc->cgx_raw = cgx_raw; - } + cgx_raw = cbw_taskc_get_cgx_raw(taskc, cgrp_id); + if (!cgx_raw) + return 0; cgx = (scx_cgroup_ctx_t *)cgx_raw; /* @@ -1782,21 +1956,10 @@ int scx_cgroup_bw_consume(u64 cgrp_id, u64 consumed_ns, u64 taskc_raw) return -EINVAL; } - /* - * Use the cached llcx if the LLC id matches; otherwise look up by - * cgx->id (avoids cgroup_get_id() pointer dereferences) and update - * the cache. - */ - if (taskc->llcx_raw && taskc->last_llc_id == llc_id) { - llcx = (scx_cgroup_llc_ctx_t *)taskc->llcx_raw; - } else { - llcx_raw = cbw_get_llc_ctx_raw_with_id(cgx->id, llc_id); - if (!llcx_raw) - return 0; - taskc->llcx_raw = llcx_raw; - taskc->last_llc_id = llc_id; - llcx = (scx_cgroup_llc_ctx_t *)llcx_raw; - } + llcx_raw = cbw_taskc_get_llcx_raw(taskc, cgx->id, llc_id); + if (!llcx_raw) + return 0; + llcx = (scx_cgroup_llc_ctx_t *)llcx_raw; accounting_out: /* @@ -1817,9 +1980,6 @@ int scx_cgroup_bw_consume(u64 cgrp_id, u64 consumed_ns, u64 taskc_raw) * bandwidth correct. */ __sync_fetch_and_add(&llcx->runtime_total, consumed_ns); - - cbw_dbg(" cgrp_id: %llu -- llc_id: %d -- consumed_ns: %llu -- llcx:runtime_total: %lld", - cgrp_id, llc_id, consumed_ns, READ_ONCE(llcx->runtime_total)); return 0; } @@ -1828,8 +1988,9 @@ int cbw_put_aside(u64 ctx, u64 vtime, u64 cgrp_id) { scx_task_common *taskc = (scx_task_common *)ctx; scx_cgroup_llc_ctx_t *llcx; - scx_atq_t *btq; int llc_id, ret; + scx_atq_t *btq; + u64 btq_vtime; /* Get the current LLC ID. */ if ((llc_id = cbw_get_current_llc_id()) < 0) { @@ -1855,6 +2016,17 @@ int cbw_put_aside(u64 ctx, u64 vtime, u64 cgrp_id) if (!btq) return -ESRCH; + /* + * Build the BTQ sort key by blending the current wall-clock time + * (upper 32 bits) with the scheduler-provided vtime (lower 32 bits). + * Tasks enqueued within the same ~4-second window compete by their + * vtime, preserving relative fairness. Once the wall-clock epoch + * advances, earlier-queued tasks take priority regardless of vtime, + * guaranteeing forward progress for any task stalled in the BTQ. + */ + btq_vtime = (scx_bpf_now() & CBW_BTQ_VTIME_UPPER_MASK) | + (vtime & CBW_BTQ_VTIME_LOWER_MASK); + ret = scx_atq_lock(btq); if (ret) { cbw_err("Failed to lock ATQ."); @@ -1869,13 +2041,12 @@ int cbw_put_aside(u64 ctx, u64 vtime, u64 cgrp_id) * for the task. This should be rare, but possible. * The spinlock turns the race into a benign one. */ - cbw_dbg("Possible double enqueue detected."); scx_atq_unlock(btq); cbw_warn("put_aside skipped: already in BTQ; cgid=%llu", cgrp_id); return 0; } - ret = scx_atq_insert_vtime_unlocked(btq, taskc, vtime); + ret = scx_atq_insert_vtime_unlocked(btq, taskc, btq_vtime); if (ret) cbw_err("Failed to insert a task to BTQ: %d", ret); @@ -1903,7 +2074,6 @@ int cbw_put_aside(u64 ctx, u64 vtime, u64 cgrp_id) __hidden int scx_cgroup_bw_put_aside(struct task_struct *p __arg_trusted, u64 ctx, u64 vtime, u64 cgrp_id) { - cbw_dbg(" [%s/%d]", p->comm, p->pid); return cbw_put_aside(ctx, vtime, cgrp_id); } @@ -1928,11 +2098,87 @@ bool cbw_has_backlogged_tasks(scx_cgroup_ctx_t *cgx) return false; } +static u32 cbw_compute_pressure(scx_cgroup_ctx_t *cgx, u32 nr_pending) +{ + /* + * Returns a CBW_PRESSURE_SCALE (1024) based value that the BPF scheduler + * uses to shorten a task's time slice: + * + * slice = (base_slice * CBW_PRESSURE_SCALE) / pressure + * + * When a cgroup is throttled, tasks that are already running hold the CPU + * for the full base slice before yielding. This delays the scheduler from + * rechecking the throttle state and causes task-stall latency. Shorter + * slices make running tasks yield more often, giving the scheduler more + * opportunities to move them to the BTQ once the budget is exhausted. + */ + u64 budget_frac, budget_pressure, backlog_pressure; + + /* + * Budget pressure + * --------------- + * Reflects how much of the newly replenished period_budget remains. When + * period_budget is close to nquota_ub the cgroup has plenty of headroom and + * pressure stays at CBW_PRESSURE_NORMAL (1024). Below CBW_PRESSURE_BUDGET_KNEE + * (25% of quota) the curve turns hyperbolic: halving the remaining budget + * doubles the pressure, so time slices shrink proportionally. This ensures + * that the last few nanoseconds of quota are consumed in small steps rather + * than one long task run that overshoots the limit. + * + * A small period_budget also indicates accumulated debt from previous + * periods (carried_debt reduced the new budget). High pressure in this + * case is correct: the cgroup already owes CPU time and should not be + * allowed to accumulate more debt through long slices. + */ + budget_frac = min((u64)max(cgx->period_budget, 0LL) * + CBW_PRESSURE_SCALE / cgx->nquota_ub, + (u64)CBW_PRESSURE_SCALE); + if (budget_frac >= CBW_PRESSURE_BUDGET_KNEE) + budget_pressure = CBW_PRESSURE_NORMAL; + else + budget_pressure = (u64)CBW_PRESSURE_NORMAL * CBW_PRESSURE_BUDGET_KNEE / + max(budget_frac, (u64)1); + + /* + * Backlog pressure + * ---------------- + * Counts tasks queued in the BTQs of all LLC domains. Each pending task + * adds CBW_PRESSURE_BACKLOG_STEP (128) to the pressure floor. A growing + * backlog means the reenqueue path cannot drain the BTQ fast enough; making + * running tasks yield sooner reduces the time any single task holds the CPU + * and allows the scheduler to interleave reenqueued tasks more quickly. + * + * @nr_pending is computed by the caller (cbw_replenish_cgroup), so we + * don't iterate the LLC list a second time here. + */ + backlog_pressure = CBW_PRESSURE_NORMAL + + CBW_PRESSURE_BACKLOG_STEP * + min(nr_pending, (u32)CBW_PRESSURE_BACKLOG_CAP); + + /* + * Two independent signals are combined by addition: + * + * pressure = (budget_pressure - NORMAL) + (backlog_pressure - NORMAL) + NORMAL + * + * Each signal contributes its excess above the NORMAL baseline. The two + * signals reflect orthogonal stress factors: low budget AND a deep queue + * is genuinely more urgent than either condition alone, so their effects + * should compound. + */ + return (u32)clamp(budget_pressure + backlog_pressure - CBW_PRESSURE_NORMAL, + (u64)CBW_PRESSURE_NORMAL, (u64)CBW_PRESSURE_MAX); +} + static bool cbw_replenish_cgroup(scx_cgroup_ctx_t *cgx, u64 now) { - s64 burst_credit = 0, debt = 0, budget; - bool period_end, was_throttled, keep_throttled = false; + s64 burst_credit, debt, budget; + bool period_end, was_throttled, keep_throttled; + u32 nr_pending; + int i; + + burst_credit = debt = 0; + keep_throttled = false; /* * If the nquota_ub is infinite, we don’t need to replenish the cgroup. @@ -1990,6 +2236,24 @@ bool cbw_replenish_cgroup(scx_cgroup_ctx_t *cgx, u64 now) budget = (s64)cgx->nquota_ub + burst_credit - debt; WRITE_ONCE(cgx->period_budget, budget); + /* + * Sum BTQ depth across LLCs to feed cbw_compute_pressure() and + * publish both cgx->nr_pending and cgx->pressure for this period. + * @nr_pending is also surfaced via cbw_dump_cgroup() so operators + * can see backlog alongside the throttle counters. + */ + nr_pending = 0; + if (cgx->has_llcx) { + bpf_for(i, 0, TOPO_NR(LLC)) { + scx_cgroup_llc_ctx_t *llcx = + cbw_get_llc_ctx_with_id(cgx->id, i); + if (llcx) + nr_pending += scx_atq_nr_queued(llcx->btq); + } + } + WRITE_ONCE(cgx->nr_pending, nr_pending); + WRITE_ONCE(cgx->pressure, cbw_compute_pressure(cgx, nr_pending)); + /* * If budget <= 0, the cgroup's debt exceeds its quota and burst for * this period, so it has no CPU time to spend. Keep it throttled so @@ -2126,6 +2390,37 @@ static struct cgroup *cbw_get_root_cgrp(void) return root; } +/** + * scx_cgroup_bw_pressure - Return the scheduling pressure for a cgroup. + * @cgrp_id: cgroup id. + * @taskc_raw: per-task context (scx_task_cgroup_bw *) cast to u64 for caching; + * pass 0 when no task context is available. + * + * Returns a 1024-scale pressure value computed at the last replenishment + * boundary. The BPF scheduler should scale a task's time slice as: + * + * slice = (base_slice * 1024) / pressure + * + * Return 1024 on error or when no throttle applies. + */ +__hidden +int scx_cgroup_bw_pressure(u64 cgrp_id, u64 taskc_raw) +{ + scx_task_cgroup_bw_t *taskc = (scx_task_cgroup_bw_t *)taskc_raw; + scx_cgroup_ctx_t *cgx; + u64 cgx_raw; + + if (cgrp_id == ROOT_CGID) + return CBW_PRESSURE_NORMAL; + + cgx_raw = cbw_taskc_get_cgx_raw(taskc, cgrp_id); + if (!cgx_raw) + return CBW_PRESSURE_NORMAL; + cgx = (scx_cgroup_ctx_t *)cgx_raw; + + return (int)READ_ONCE(cgx->pressure); +} + /* * A handler function for the accounting timer. @@ -2134,7 +2429,7 @@ static int accounting_timerfn(void *map, int *key, struct bpf_timer *timer) { struct cgroup *root_cgrp; - u64 now, next_interval = CBW_ACCOUNTING_PERIOD_MAX; + u64 next_interval = CBW_ACCOUNTING_PERIOD_MAX; int ret; /* @@ -2151,9 +2446,6 @@ int accounting_timerfn(void *map, int *key, struct bpf_timer *timer) if (unlikely(cbw_top_half_running())) goto release_out; - now = scx_bpf_now(); - cbw_dbg("at %llu", now); - cbw_update_runtime_total_sloppy(root_cgrp); next_interval = cbw_throttle_cgroups(root_cgrp); smp_mb(); @@ -2193,7 +2485,6 @@ int replenish_timerfn(void *map, int *key, struct bpf_timer *timer) */ now = scx_bpf_now(); cbw_top_half_begin(); - cbw_dbg("at %llu", now); /* * Update the runtime total before replenishing budgets. @@ -2287,7 +2578,6 @@ int replenish_timerfn(void *map, int *key, struct bpf_timer *timer) * the burst time. However, relaxing some accuracy in burst time * calculation has more benefits than drawbacks. */ - cbw_dbg("Start replenish %llu cgroups.", cbw_nr_cgroups); nr_throttled = 0; bpf_for(i, 0, cbw_nr_cgroups) { ids = MEMBER_VPTR(cbw_cgroup_ids, [i]); @@ -2302,13 +2592,11 @@ int replenish_timerfn(void *map, int *key, struct bpf_timer *timer) */ cur_cgrp = bpf_cgroup_from_id(ids[0]); if (!cur_cgrp) { - cbw_dbg("Failed to fetch a cgroup pointer: cgid%llu", ids[0]); goto offline_cgroup; } cur_cgx = cbw_get_cgroup_ctx(cur_cgrp); if (!cur_cgx) { - cbw_dbg("Failed to lookup a cgroup ctx: cgid%llu", ids[0]); bpf_cgroup_release(cur_cgrp); goto offline_cgroup; } @@ -2470,7 +2758,6 @@ int cbw_drain_btq_batch(scx_cgroup_ctx_t *cgx, */ scx_cgroup_bw_enqueue_cb((u64)taskc); - cbw_dbg("cgid%llu", cgx->id); } return i; @@ -2492,7 +2779,6 @@ int cbw_reenqueue_cgroup(struct cgroup *cgrp, scx_cgroup_ctx_t *cgx, */ if (!cgx->has_llcx) return false; - cbw_dbg("cgid%llu", cgrp_id); bpf_for(i, 0, TOPO_NR(LLC)) { idx = (nuance + i) % TOPO_NR(LLC); @@ -2576,7 +2862,6 @@ int scx_cgroup_bw_reenqueue(void) * Note that we intentionally ignore the error to reenqueue all the * tasks, ensuring it always returns 0. */ - cbw_dbg(); nuance = bpf_get_prandom_u32(); nr_tcgs = backlog_stat.nr_throttled_cgroups; bpf_for(i, 0, nr_tcgs) { @@ -2601,7 +2886,6 @@ int scx_cgroup_bw_reenqueue(void) /* Never tear down root; see cbw_get_root_cgrp(). */ if (cur_cgrp_id == ROOT_CGID) continue; - cbw_dbg("Failed to fetch a cgroup pointer: %llu", ids[0]); /* * This cgroup is already offline: its kernfs node is @@ -2752,24 +3036,17 @@ int scx_cgroup_bw_move(struct task_struct *p __arg_trusted, u64 taskc, struct cgroup *from __arg_trusted, struct cgroup *to __arg_trusted) { - volatile scx_task_cgroup_bw_t *tc; /* Add `volatile` to work around the verifier error */ + scx_task_cgroup_bw_t *tc; int ret; scx_arena_subprog_init(); /* * Invalidate the per-task cache: cgx_raw and llcx_raw belong to the * old cgroup and will be repopulated on the next throttle/consume call. - * - * Use atomic exchanges instead of plain stores: LLVM folds constant - * stores into base+offset addressing and omits addr_space_cast for the - * arena pointer, which the BPF verifier rejects. Atomics always emit - * addr_space_cast for the base register regardless of offset. */ tc = (scx_task_cgroup_bw_t *)taskc; - if (tc) { - __sync_lock_test_and_set(&tc->cgx_raw, 0); - __sync_lock_test_and_set(&tc->llcx_raw, 0); - } + if (tc) + cbw_taskc_invalidate(tc); /* * If a task is throttled, remove it from the @from cgroup, @@ -2796,8 +3073,17 @@ int scx_cgroup_bw_move(struct task_struct *p __arg_trusted, u64 taskc, return ret; } +#define cbw_dump_line(mode, fmt, args...) \ + do { \ + if ((mode) == SCX_CGROUP_BW_DUMP_SCX) \ + scx_bpf_dump(fmt "\n", ##args); \ + else \ + bpf_printk(fmt, ##args); \ + } while (0) + static __noinline -int cbw_dump_cgroup(struct cgroup *cgrp __arg_trusted, bool indent) +int cbw_dump_cgroup(struct cgroup *cgrp __arg_trusted, bool indent, + enum scx_cgroup_bw_dump_mode mode) { static const char indent_strs[][64] = { "", @@ -2847,17 +3133,16 @@ int cbw_dump_cgroup(struct cgroup *cgrp __arg_trusted, bool indent) cgx = cbw_get_cgroup_ctx(cgrp); if (!cgx) { - cbw_dbg("Failed to lookup a cgroup context: %llu", cgroup_get_id(cgrp)); return -ESRCH; } indent_str = indent_strs[ clamp((u32)cgrp->level, 0, indent_max - 1) ]; bpf_probe_read_kernel_str(name, sizeof(name), BPF_CORE_READ(cgrp->kn, name)); - bpf_printk("%s +-- %s (id: %llu, level: %d)", indent_str, + cbw_dump_line(mode, "%s +-- %s (id: %llu, level: %d)", indent_str, name, cgroup_get_id(cgrp), (u32)cgrp->level); - if (cgx->nquota_ub == CBW_RUNTUME_INF) + if (cgx->nquota == CBW_RUNTUME_INF) return 0; if (cgx->has_llcx) { @@ -2869,18 +3154,18 @@ int cbw_dump_cgroup(struct cgroup *cgrp __arg_trusted, bool indent) } } - bpf_printk("%s \\_ quota: %llu/%llu/%llu, period: %llu, burst: %llu", indent_str, + cbw_dump_line(mode, "%s \\_ quota: %llu, period: %llu, burst: %llu", indent_str, cgx->quota, cgx->period, cgx->burst); - bpf_printk("%s \\_ nquota: %llu, nquota_ub: %llu, has_llcx: %d", indent_str, + cbw_dump_line(mode, "%s \\_ nquota: %llu, nquota_ub: %llu, has_llcx: %d", indent_str, cgx->nquota, cgx->nquota_ub, cgx->has_llcx); - bpf_printk("%s \\_ is_throttled: %d, nr_throttled_periods: %d/%d, nr_throttled_tasks: %d", indent_str, + cbw_dump_line(mode, "%s \\_ is_throttled: %d, nr_throttled_periods: %d/%d, nr_throttled_tasks: %d", indent_str, cgx->is_throttled, cgx->nr_throttled_periods, READ_ONCE(cbw_backlog_stat.rp_seq) / 2, nr_throttled_tasks); - bpf_printk("%s \\_ period_budget: %lld, burst_remaining: %lld", indent_str, - cgx->period_budget, cgx->burst_remaining); - bpf_printk("%s \\_ runtime_total_sloppy: %lld, runtime_total_last: %lld", indent_str, - cgx->runtime_total_sloppy, cgx->runtime_total_last); + cbw_dump_line(mode, "%s \\_ period_budget: %lld, burst_remaining: %lld, avg_consumption_rate: %llu", indent_str, + cgx->period_budget, cgx->burst_remaining, cgx->avg_consumption_rate); + cbw_dump_line(mode, "%s \\_ pressure: %u, nr_pending: %u, runtime_total_sloppy: %lld, runtime_total_last: %lld", indent_str, + cgx->pressure, cgx->nr_pending, cgx->runtime_total_sloppy, cgx->runtime_total_last); return 0; } @@ -2895,18 +3180,23 @@ int cbw_dump_cgroup(struct cgroup *cgrp __arg_trusted, bool indent) * get more accurate information. Otherwise, dump the currently collected * snapshot of runtime values. * @indent: If true, indent the output. Otherwise, do not indent the output. + * @mode: Output sink (see enum scx_cgroup_bw_dump_mode). * * Return 0 for success, -errno for failure. */ __hidden -int scx_cgroup_bw_dump(u64 cgrp_id, bool descendent, bool accurate, bool indent) +int scx_cgroup_bw_dump(u64 cgrp_id, bool descendent, bool accurate, bool indent, + enum scx_cgroup_bw_dump_mode mode) { struct cgroup_subsys_state *start_css, *pos; struct cgroup *start_cgrp, *cur_cgrp; + /* Resolve the hard-coded root cgroup id. */ + if (cgrp_id == 1) + cgrp_id = ROOT_CGID; + start_cgrp = bpf_cgroup_from_id(cgrp_id); if (!start_cgrp) { - cbw_dbg("Failed to fetch a cgroup pointer: cgid%llu", cgrp_id); return -ESRCH; } @@ -2914,7 +3204,7 @@ int scx_cgroup_bw_dump(u64 cgrp_id, bool descendent, bool accurate, bool indent) cbw_update_runtime_total_sloppy(start_cgrp); if (!descendent) { - cbw_dump_cgroup(start_cgrp, indent); + cbw_dump_cgroup(start_cgrp, indent, mode); goto release_out; } @@ -2922,7 +3212,7 @@ int scx_cgroup_bw_dump(u64 cgrp_id, bool descendent, bool accurate, bool indent) start_css = &start_cgrp->self; bpf_for_each(css, pos, start_css, BPF_CGROUP_ITER_DESCENDANTS_PRE) { cur_cgrp = pos->cgroup; - cbw_dump_cgroup(cur_cgrp, indent); + cbw_dump_cgroup(cur_cgrp, indent, mode); } bpf_rcu_read_unlock(); diff --git a/scheds/include/lib/atq.h b/scheds/include/lib/atq.h index b6dc2b0edc..24523154dd 100644 --- a/scheds/include/lib/atq.h +++ b/scheds/include/lib/atq.h @@ -72,11 +72,14 @@ int scx_atq_lock(scx_atq_t __arg_arena *atq) * just bails, leaving the MCS chain with stale ->next links and any * waiters queued behind it stuck on their own bounded spins. * Subsequent acquires race against an inconsistent queue; retrying - * is unsafe. Treat the timeout as a fatal scheduler error so the - * system tears down cleanly. + * is unsafe. Tear down via scx_bpf_exit(SCX_ECODE_ACT_RESTART, ...) + * so user-space orchestration can respawn the scheduler -- a fresh + * load reinitialises the MCS state. */ - if (ret == -ETIMEDOUT) - scx_bpf_error("scx_atq: arena_spin_lock timed out"); + if (ret == -ETIMEDOUT) { + scx_bpf_exit(SCX_ECODE_ACT_RESTART, + "scx_atq: arena_spin_lock timed out"); + } return ret; } diff --git a/scheds/include/lib/cgroup.h b/scheds/include/lib/cgroup.h index 713ba6c9cc..b718c2d3f2 100644 --- a/scheds/include/lib/cgroup.h +++ b/scheds/include/lib/cgroup.h @@ -148,6 +148,21 @@ int scx_cgroup_bw_reenqueue(void); */ int scx_cgroup_bw_cancel(u64 taskc); +/** + * scx_cgroup_bw_pressure - Return the scheduling pressure for a cgroup. + * @cgrp_id: cgroup id. + * @taskc: per-task context (scx_task_cgroup_bw *) cast to u64 for caching; + * pass 0 when no task context is available. + * + * Returns a 1024-scale pressure value computed at the last replenishment + * boundary. The BPF scheduler should scale a task's time slice as: + * + * slice = (base_slice * 1024) / pressure + * + * Return 1024 on error or when no throttle applies. + */ +int scx_cgroup_bw_pressure(u64 cgrp_id, u64 taskc); + /** * REGISTER_SCX_CGROUP_BW_ENQUEUE_CB - Register an enqueue callback. * @eqcb: A function name with a prototype of @@ -210,6 +225,24 @@ int scx_cgroup_bw_move(struct task_struct *p __arg_trusted, u64 taskc, struct cgroup *from __arg_trusted, struct cgroup *to __arg_trusted); +/** + * Output sink selector for scx_cgroup_bw_dump(). + * + * SCX_CGROUP_BW_DUMP_PRINTK: route lines through bpf_printk(); the output + * appears in the kernel trace pipe (e.g. trace_pipe / bpftool prog + * tracelog). Usable from any BPF context. + * + * SCX_CGROUP_BW_DUMP_SCX: route lines through scx_bpf_dump(); the output + * becomes part of the SCX dump buffer surfaced on scheduler error/exit + * or via the SCX dump-on-error machinery. MUST only be called from an + * ops.dump*() callback -- the BPF verifier rejects scx_bpf_dump_bstr() + * from other contexts. + */ +enum scx_cgroup_bw_dump_mode { + SCX_CGROUP_BW_DUMP_PRINTK = 0, + SCX_CGROUP_BW_DUMP_SCX = 1, +}; + /** * scx_cgroup_bw_dump - Dump the cgroup status * @@ -220,10 +253,12 @@ int scx_cgroup_bw_move(struct task_struct *p __arg_trusted, u64 taskc, * get more accurate information. Otherwise, dump the currently collected * snapshot of runtime values. * @indent: If true, indent the output. Otherwise, do not indent the output. + * @mode: Output sink (see enum scx_cgroup_bw_dump_mode). * * Return 0 for success, -errno for failure. */ -int scx_cgroup_bw_dump(u64 cgrp_id, bool descendent, bool accurate, bool indent); +int scx_cgroup_bw_dump(u64 cgrp_id, bool descendent, bool accurate, bool indent, + enum scx_cgroup_bw_dump_mode mode); /** * Per-task context for CPU bandwidth control. diff --git a/scheds/rust/scx_lavd/src/bpf/main.bpf.c b/scheds/rust/scx_lavd/src/bpf/main.bpf.c index 0f1fd802b5..ea972bdd07 100644 --- a/scheds/rust/scx_lavd/src/bpf/main.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/main.bpf.c @@ -285,23 +285,41 @@ static void advance_cur_logical_clk(struct task_struct *p) } } +static u32 get_cgroup_pressure(task_ctx *taskc) +{ + if (!enable_cpu_bw) + return LAVD_SCALE; + + return scx_cgroup_bw_pressure(taskc->cgrp_id, (u64)taskc); +} + static u64 calc_time_slice(task_ctx *taskc, struct cpu_ctx *cpuc) { + u64 slice_wall; + u32 pressure; + /* * Calculate the time slice of @taskc to run on @cpuc. */ if (!taskc || !cpuc) return LAVD_SLICE_MAX_NS_DFL; + /* + * Get the cgroup pressure to scale down the time slice when the + * cgroup is throttled. A pressure greater than LAVD_SCALE reduces + * the slice proportionally, making tasks yield the CPU more often. + */ + pressure = get_cgroup_pressure(taskc); + /* * If pinned_slice_ns is enabled and there are pinned tasks waiting * to run on this CPU, unconditionally reduce the time slice for * all tasks to ensure pinned tasks can run promptly. */ if (pinned_slice_ns && cpuc->nr_pinned_tasks) { - taskc->slice_wall = min(pinned_slice_ns, sys_stat.slice_wall); + slice_wall = min(pinned_slice_ns, sys_stat.slice_wall); reset_task_flag(taskc, LAVD_FLAG_SLICE_BOOST); - return taskc->slice_wall; + goto out; } /* @@ -316,8 +334,12 @@ static u64 calc_time_slice(task_ctx *taskc, struct cpu_ctx *cpuc) * However, if there are pinned tasks waiting to run on this CPU, * we do not boost the task's time slice to avoid delaying the pinned * task that cannot be run on another CPU. + * + * Also, if a cgroup is under pressure (e.g., throttled), do not boost + * time slice. */ - if (!no_slice_boost && !cpuc->nr_pinned_tasks && + if (!no_slice_boost && pressure <= LAVD_SCALE && + !cpuc->nr_pinned_tasks && (taskc->avg_runtime_wall >= sys_stat.slice_wall)) { /* * When the system is not heavily loaded, so it can serve all @@ -339,10 +361,9 @@ static u64 calc_time_slice(task_ctx *taskc, struct cpu_ctx *cpuc) * bit longer than average, can still finish the job. */ u64 s = taskc->avg_runtime_wall + LAVD_SLICE_BOOST_BONUS; - taskc->slice_wall = clamp(s, slice_min_ns, - LAVD_SLICE_BOOST_MAX); + slice_wall = clamp(s, slice_min_ns, LAVD_SLICE_BOOST_MAX); set_task_flag(taskc, LAVD_FLAG_SLICE_BOOST); - return taskc->slice_wall; + goto out; } /* @@ -356,12 +377,12 @@ static u64 calc_time_slice(task_ctx *taskc, struct cpu_ctx *cpuc) u64 b = (sys_stat.slice_wall * taskc->lat_cri) / (sys_stat.avg_lat_cri + 1); u64 s = sys_stat.slice_wall + b; - taskc->slice_wall = clamp(s, slice_min_ns, - min(taskc->avg_runtime_wall, - sys_stat.slice_wall * 2)); + slice_wall = clamp(s, slice_min_ns, + min(taskc->avg_runtime_wall, + sys_stat.slice_wall * 2)); set_task_flag(taskc, LAVD_FLAG_SLICE_BOOST); - return taskc->slice_wall; + goto out; } } @@ -369,9 +390,11 @@ static u64 calc_time_slice(task_ctx *taskc, struct cpu_ctx *cpuc) * If slice boost is either not possible, not necessary, or not * eligible, assign the regular time slice. */ - taskc->slice_wall = sys_stat.slice_wall; + slice_wall = sys_stat.slice_wall; reset_task_flag(taskc, LAVD_FLAG_SLICE_BOOST); - return taskc->slice_wall; + +out: + return taskc->slice_wall = max((slice_wall * LAVD_SCALE) / pressure, 1); } static void update_stat_for_running(struct task_struct *p, @@ -2031,10 +2054,12 @@ s32 BPF_STRUCT_OPS(lavd_exit_task, struct task_struct *p, void BPF_STRUCT_OPS(lavd_dump, struct scx_dump_ctx *dctx) { /* - * Dump the cpu.max status of the entire cgroup hierarchy. + * Dump the cpu.max status of the entire cgroup hierarchy + * through bpf_printk. */ if (enable_cpu_bw) { - scx_cgroup_bw_dump(1, true, true, true); + scx_cgroup_bw_dump(1, true, true, true, + SCX_CGROUP_BW_DUMP_PRINTK); } } @@ -2075,6 +2100,11 @@ void BPF_STRUCT_OPS(lavd_dump_task, struct scx_dump_ctx *dctx, cgrp_name, taskc->cgrp_id, (cgroup_throttled) ? "throttled" : "not throttled", (task_throttled) ? "throttled" : "not throttled"); + + if (enable_cpu_bw && (cgroup_throttled || task_throttled)) { + scx_cgroup_bw_dump(taskc->cgrp_id, false, true, false, + SCX_CGROUP_BW_DUMP_SCX); + } } static s32 init_cpdoms(u64 now) diff --git a/scripts/cpu_max_bench.ini b/scripts/cpu_max_bench.ini new file mode 100644 index 0000000000..95f1294708 --- /dev/null +++ b/scripts/cpu_max_bench.ini @@ -0,0 +1,441 @@ +# cpu_max_bench.py configuration file +# +# Each section defines one benchmark run. +# Runs with the same (cgroup_depth, quota, load_factor) are placed in the +# same report group and share a single CPU-utilisation graph. +# +# Keys and their defaults: +# cgroup_depth = 32 +# quota = max # "max" or a float (percent of nproc) +# load_factor = 100 # workers = nproc * load_factor / 100 +# duration = 60 # seconds +# scheduler = eevdf # eevdf | scx_lavd +# scx_path = /usr/bin/scx_lavd +# scx_args = --performance --enable-cpu-bw +# +# Usage: +# sudo python3 scripts/cpu_max_bench.py -o results/ scripts/cpu_max_bench.ini + +# ================================================================== +# Group 0 -- Root cgroup: workload runs directly in the system root +# cgroup with no per-run hierarchy at all (cgroup_depth=0). +# cpu.stat at the root reports SYSTEM-WIDE usage, so the load +# stress-ng generates dominates the measurement. Establishes a +# "no-cgroup-overhead" baseline for comparison. +# group key: cgroup_depth=0, quota=max, load_factor=100 +# Compares: eevdf, scx_lavd w/ cpu-bw, scx_lavd w/o cpu-bw +# ================================================================== + +[root_eevdf] +cgroup_depth = 0 +quota = max +load_factor = 100 +duration = 60 +scheduler = eevdf + +[root_lavd] +cgroup_depth = 0 +quota = max +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance + +[root_lavd_bw] +cgroup_depth = 0 +quota = max +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ================================================================== +# Group 1 -- Baseline: no cpu.max enforcement +# group key: cgroup_depth=32, quota=max, load_factor=100 +# Compares: eevdf, scx_lavd w/ cpu-bw, scx_lavd w/o cpu-bw +# ================================================================== + +[baseline_eevdf] +cgroup_depth = 32 +quota = max +load_factor = 100 +duration = 60 +scheduler = eevdf + +[baseline_lavd] +cgroup_depth = 32 +quota = max +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance + +[baseline_lavd_bw] +cgroup_depth = 32 +quota = max +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ================================================================== +# Group 2 -- Various CPU loads with cpu.max enforced (depth=32) +# Each load_factor is its own group: (depth=32, quota=100%, load=X) +# Compares: eevdf vs. scx_lavd w/ cpu-bw at each load level +# ================================================================== + +[load_005pct_eevdf] +cgroup_depth = 32 +quota = 100 +load_factor = 5 +duration = 60 +scheduler = eevdf + +[load_005pct_lavd_bw] +cgroup_depth = 32 +quota = 100 +load_factor = 5 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[load_050pct_eevdf] +cgroup_depth = 32 +quota = 100 +load_factor = 50 +duration = 60 +scheduler = eevdf + +[load_050pct_lavd_bw] +cgroup_depth = 32 +quota = 100 +load_factor = 50 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[load_100pct_eevdf] +cgroup_depth = 32 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = eevdf + +[load_100pct_lavd_bw] +cgroup_depth = 32 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[load_125pct_eevdf] +cgroup_depth = 32 +quota = 100 +load_factor = 125 +duration = 60 +scheduler = eevdf + +[load_125pct_lavd_bw] +cgroup_depth = 32 +quota = 100 +load_factor = 125 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[load_200pct_eevdf] +cgroup_depth = 32 +quota = 100 +load_factor = 200 +duration = 60 +scheduler = eevdf + +[load_200pct_lavd_bw] +cgroup_depth = 32 +quota = 100 +load_factor = 200 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[load_1000pct_eevdf] +cgroup_depth = 32 +quota = 100 +load_factor = 1000 +duration = 60 +scheduler = eevdf + +[load_1000pct_lavd_bw] +cgroup_depth = 32 +quota = 100 +load_factor = 1000 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ================================================================== +# Group 3 -- Various cgroup depths with cpu.max enforced (load=100) +# Each depth is its own group: (depth=X, quota=100%, load=100) +# Note: depth=32 is already covered by load_100pct_* in Group 2 +# (same group key -- they will be merged in the report). +# Compares: eevdf vs. scx_lavd w/ cpu-bw at each depth +# ================================================================== + +[depth_01_eevdf] +cgroup_depth = 1 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = eevdf + +[depth_01_lavd_bw] +cgroup_depth = 1 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[depth_08_eevdf] +cgroup_depth = 8 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = eevdf + +[depth_08_lavd_bw] +cgroup_depth = 8 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[depth_16_eevdf] +cgroup_depth = 16 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = eevdf + +[depth_16_lavd_bw] +cgroup_depth = 16 +quota = 100 +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# depth=32 / load=100: see load_100pct_* in Group 2 above + +# ================================================================== +# Group 4 -- 50% quota (half of nproc), varying load factors (depth=32) +# Each load_factor is its own group: (depth=32, quota=50%, load=X) +# Compares: eevdf vs. scx_lavd w/ cpu-bw at each load level +# ================================================================== + +[q50pct_load_010_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 10 +duration = 60 +scheduler = eevdf + +[q50pct_load_010_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 10 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_020_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 20 +duration = 60 +scheduler = eevdf + +[q50pct_load_020_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 20 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_030_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 30 +duration = 60 +scheduler = eevdf + +[q50pct_load_030_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 30 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_040_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 40 +duration = 60 +scheduler = eevdf + +[q50pct_load_040_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 40 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_050_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 50 +duration = 60 +scheduler = eevdf + +[q50pct_load_050_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 50 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_060_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 60 +duration = 60 +scheduler = eevdf + +[q50pct_load_060_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 60 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_070_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 70 +duration = 60 +scheduler = eevdf + +[q50pct_load_070_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 70 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_080_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 80 +duration = 60 +scheduler = eevdf + +[q50pct_load_080_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 80 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_090_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 90 +duration = 60 +scheduler = eevdf + +[q50pct_load_090_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 90 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw + +# ------------------------------------------------------------------ + +[q50pct_load_100_eevdf] +cgroup_depth = 32 +quota = 50 +load_factor = 100 +duration = 60 +scheduler = eevdf + +[q50pct_load_100_lavd_bw] +cgroup_depth = 32 +quota = 50 +load_factor = 100 +duration = 60 +scheduler = scx_lavd +scx_path = /usr/bin/scx_lavd +scx_args = --performance --enable-cpu-bw diff --git a/scripts/cpu_max_bench.py b/scripts/cpu_max_bench.py new file mode 100755 index 0000000000..f14c419408 --- /dev/null +++ b/scripts/cpu_max_bench.py @@ -0,0 +1,1080 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (c) 2026 Meta Platforms, Inc. and affiliates. +# Author: Changwoo Min +# +# cpu_max_bench.py - Benchmark for measuring cpu.max cgroup bandwidth overhead +# +# Runs stress-ng --cpu inside a deep cgroup hierarchy with cpu.max enforced at +# every level, measuring kernel-mode cycle overhead via "perf stat -a". +# +# Usage: +# sudo ./cpu_max_bench.py [OPTIONS] [CONFIG_FILE] +# +# See --help for full option list and CONFIG_FILE format. + +import argparse +import atexit +import configparser +import fnmatch +import os +import re +import signal +import subprocess +import sys +import threading +import time +from datetime import datetime +from pathlib import Path + +try: + import matplotlib + matplotlib.use('Agg') + import matplotlib.pyplot as plt + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + +NPROC = os.cpu_count() or 1 +CGROUP_ROOT = Path('/sys/fs/cgroup') +PERIOD_US = 100_000 # 100 ms cgroup period +BENCH_ID_PREFIX = 'cpu_max_bench' + +# Sleep durations used inside run_benchmark(); kept as constants so the +# estimated run-time printed at startup stays in sync with actual behaviour. +SLEEP_PERF_ARM = 0.3 # let perf arm before stress-ng starts +SLEEP_CGROUP_MOVE = 0.5 # let stress-ng spawn workers before cgroup move +SLEEP_SCX_STARTUP = 20.0 # wait for scx_lavd to become active + +# Per-process monotonic counter so two run_benchmark() calls in the same +# millisecond still produce distinct cgroup names. Combined with PID and +# wall-clock millisecond it makes the bench_id collision-free. +_bench_seq = 0 + + +def _next_bench_id() -> str: + global _bench_seq + _bench_seq += 1 + return f'{BENCH_ID_PREFIX}_{os.getpid()}_{int(time.time() * 1000)}_{_bench_seq}' + + +# Cleanup registry: every CgroupManager that is currently set up registers +# itself here so that a SIGTERM / SIGINT / unhandled exception triggers a +# best-effort teardown. This is the safety net for hard-exits where the +# normal try/finally in run_benchmark does not get a chance to run. +_active_cleanups: list = [] + + +def _register_cleanup(cg) -> None: + _active_cleanups.append(cg) + + +def _unregister_cleanup(cg) -> None: + try: + _active_cleanups.remove(cg) + except ValueError: + pass + + +def _run_emergency_cleanup() -> None: + """Tear down any cgroups still registered. Called from atexit and from + the SIGTERM/SIGINT handler installed in main().""" + while _active_cleanups: + cg = _active_cleanups.pop() + try: + cg.teardown() + except Exception as exc: + print(f'emergency teardown of {cg.leaf}: {exc}', file=sys.stderr) + + +def _signal_handler(signum, frame) -> None: + print(f'\nReceived signal {signum}; cleaning up cgroups...', + file=sys.stderr) + _run_emergency_cleanup() + sys.exit(128 + signum) + + +atexit.register(_run_emergency_cleanup) + + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +class Config: + """Parameters for one benchmark run.""" + + def __init__(self, name, depth=32, quota=None, load_factor=100, + duration=60, scheduler='eevdf', + scx_path='/usr/bin/scx_lavd', + scx_args='--performance --enable-cpu-bw'): + self.name = name + self.depth = int(depth) + # quota: None -> "max" (unlimited) + # float -> percent of NPROC; e.g., 100 == NPROC CPUs, + # 50 == NPROC/2 CPUs. + self.quota = quota + self.load_factor = int(load_factor) + self.duration = int(duration) + self.scheduler = scheduler + self.scx_path = scx_path + self.scx_args = scx_args + + # -- derived properties -------------------------------------------------- + + @property + def workers(self): + return max(1, int(NPROC * self.load_factor / 100)) + + @property + def quota_cpus(self): + """Quota expressed in CPUs (percent of NPROC), or None for "max".""" + if self.quota is None: + return None + return self.quota * NPROC / 100.0 + + @property + def quota_us(self): + """Quota in microseconds per PERIOD_US period, or None for "max".""" + if self.quota is None: + return None + return int(self.quota_cpus * PERIOD_US) + + @property + def quota_str(self): + if self.quota is None: + return 'max' + return f'{self.quota:g}%' + + @property + def group_key(self): + """Configurations that share these three values belong to one report group.""" + return (self.depth, self.quota, self.load_factor) + + +# --------------------------------------------------------------------------- +# Result container +# --------------------------------------------------------------------------- + +class BenchmarkResult: + """All data collected from a single benchmark run.""" + + def __init__(self, config: Config): + self.config = config + # perf stat counters + self.cycles: int = 0 + self.cycles_k: int = 0 + self.cache_misses: int = 0 + self.stalled_cycles_backend: int = 0 + self.instructions: int = 0 + # stress-ng summary + self.bogo_ops: int = 0 + self.bogo_ops_per_sec: float = 0.0 + # per-second CPU utilisation from cgroup cpu.stat + self.cpu_util_samples: list = [] # [(elapsed_s, cpu_used_cpus), ...] + # raw text from each tool + self.perf_raw: str = '' + self.stress_ng_raw: str = '' + # directory where per-run files are saved + self.output_dir: Path = Path('.') + + @property + def overhead_cpus(self) -> float: + """Kernel-mode overhead expressed as equivalent number of CPUs.""" + if self.cycles == 0: + return 0.0 + return (self.cycles_k / self.cycles) * NPROC + + +# --------------------------------------------------------------------------- +# Cgroup hierarchy management +# --------------------------------------------------------------------------- + +class CgroupManager: + """Creates and tears down a linear cgroup chain of the requested depth.""" + + def __init__(self, config: Config, bench_id: str): + self.config = config + # Per-run cgroup chain placed directly under the system root, so + # that cgroup_depth == N means the leaf is at kernel cgrp->level N + # (system root is level 0). cgroup_depth = 0 is a special case: + # no per-run cgroup is created and the workload runs in the system + # root cgroup itself, used as a "no-cgroup-overhead" baseline. + self._run_root = CGROUP_ROOT / bench_id + self._all_levels = self._build_level_paths() + self.leaf = self._all_levels[-1] if self._all_levels else CGROUP_ROOT + + def _build_level_paths(self) -> list: + """Return list of Path objects from run root down to the leaf. + + depth=0 -> empty chain; the workload runs in the system root. + depth=N (N >= 1) -> chain of N nodes, leaf at kernel level N. + """ + if self.config.depth <= 0: + return [] + paths = [self._run_root] + for i in range(1, self.config.depth): + paths.append(paths[-1] / f'l{i}') + return paths + + @staticmethod + def _write(path: Path, content: str): + try: + path.write_text(content + '\n') + except OSError as exc: + _warn(f'write {content!r} -> {path}: {exc}') + + def setup(self): + """Enable the cpu controller and create the full cgroup chain.""" + # Enable cpu at the system root (idempotent) + self._write(CGROUP_ROOT / 'cgroup.subtree_control', '+cpu') + + # depth=0: no per-run cgroup; cpu.max on the system root is left + # untouched. Any non-"max" quota in the config is silently + # ignored in this mode. + if not self._all_levels: + return + + quota_content = ( + f'max {PERIOD_US}' + if self.config.quota is None + else f'{self.config.quota_us} {PERIOD_US}' + ) + + for idx, path in enumerate(self._all_levels): + path.mkdir(exist_ok=True) + self._write(path / 'cpu.max', quota_content) + # Enable cpu controller for children unless this is the leaf + if idx < len(self._all_levels) - 1: + self._write(path / 'cgroup.subtree_control', '+cpu') + + def move_pid(self, pid: int): + """Assign a process to the leaf cgroup.""" + self._write(self.leaf / 'cgroup.procs', str(pid)) + + def read_usage_usec(self) -> int: + """Return cumulative CPU usage (us) from the leaf cgroup. + + For depth=0 the leaf is the system root; cpu.stat there reports + SYSTEM-WIDE usage (not isolated to the workload). Acceptable + for the root_* baseline as long as stress-ng dominates the load. + """ + try: + for line in (self.leaf / 'cpu.stat').read_text().splitlines(): + if line.startswith('usage_usec'): + return int(line.split()[1]) + except (OSError, ValueError): + pass + return 0 + + def teardown(self): + """Remove the per-run cgroup subtree (leaf to root). + + Steps: + 1. Write 1 to leaf/cgroup.kill to terminate any leftover + processes (cgroupv2 feature, kernel >= 5.14). + 2. Wait briefly for the leaf to become unpopulated so that + rmdir() sees an empty cgroup; otherwise rmdir() fails with + EBUSY and the cgroup leaks across runs. + 3. rmdir() bottom-up. Any rmdir failure is logged loudly so + that a leaked cgroup never goes unnoticed. + + depth=0 has no chain to remove. + """ + if not self._all_levels: + return + leaf = self.leaf + try: + (leaf / 'cgroup.kill').write_text('1\n') + except OSError: + # cgroup.kill missing or write failed (e.g. older kernel); + # fall back to relying on already-exited workers. + pass + self._wait_unpopulated(leaf, timeout=30.0) + for path in reversed(self._all_levels): + try: + path.rmdir() + except OSError as exc: + _warn(f'rmdir({path}) failed: {exc}') + + @staticmethod + def _wait_unpopulated(path: Path, timeout: float): + """Spin on cgroup.events until populated == 0 or timeout.""" + deadline = time.monotonic() + timeout + events_path = path / 'cgroup.events' + while time.monotonic() < deadline: + try: + content = events_path.read_text() + except OSError: + return + if 'populated 0' in content.splitlines(): + return + time.sleep(0.05) + + +# --------------------------------------------------------------------------- +# Scheduler management +# --------------------------------------------------------------------------- + +class SchedulerManager: + """Ensures the requested CPU scheduler is active for the benchmark.""" + + def __init__(self, config: Config): + self.config = config + self._proc = None + + def setup(self): + if self.config.scheduler == 'eevdf': + self._kill_scx_lavd() + elif self.config.scheduler == 'scx_lavd': + self._start_scx_lavd() + + def teardown(self): + if self._proc is not None: + self._proc.terminate() + try: + self._proc.wait(timeout=5) + except subprocess.TimeoutExpired: + self._proc.kill() + self._proc = None + + def _kill_scx_lavd(self): + try: + r = subprocess.run(['pgrep', '-x', 'scx_lavd'], + capture_output=True, text=True) + for pid_s in r.stdout.split(): + try: + os.kill(int(pid_s), signal.SIGTERM) + except (ProcessLookupError, ValueError): + pass + if r.stdout.strip(): + time.sleep(1) + except FileNotFoundError: + pass + + def _start_scx_lavd(self): + self._kill_scx_lavd() + cmd = [self.config.scx_path] + self.config.scx_args.split() + self._proc = subprocess.Popen(cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + time.sleep(SLEEP_SCX_STARTUP) + if self._proc.poll() is not None: + raise RuntimeError( + f'scx_lavd exited early (code {self._proc.returncode})') + + +# --------------------------------------------------------------------------- +# Per-second CPU utilisation monitor +# --------------------------------------------------------------------------- + +class CpuUtilMonitor: + """Reads cgroup cpu.stat every second and records utilisation samples.""" + + def __init__(self, cg: CgroupManager): + self._cg = cg + self.samples: list = [] # [(elapsed_s, cpu_used_cpus)] + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + def start(self): + self._stop.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def stop(self): + self._stop.set() + if self._thread: + self._thread.join(timeout=5) + + def _run(self): + prev_usec = self._cg.read_usage_usec() + prev_mono = time.monotonic() + elapsed = 0.0 + + while not self._stop.wait(1.0): + cur_usec = self._cg.read_usage_usec() + cur_mono = time.monotonic() + delta_usec = cur_usec - prev_usec + delta_sec = cur_mono - prev_mono + elapsed += delta_sec + if delta_sec > 0: + cpu_used = (delta_usec / 1_000_000) / delta_sec + self.samples.append((elapsed, cpu_used)) + prev_usec = cur_usec + prev_mono = cur_mono + + +# --------------------------------------------------------------------------- +# Output parsers +# --------------------------------------------------------------------------- + +_PERF_NUM_RE = re.compile( + r'^\s*([\d,]+(?:\.\d+)?)\s+' # numeric value (possibly with commas) + r'([\w:/-]+)' # event name +) + +def parse_perf_output(text: str) -> dict: + """Extract counter values from ``perf stat`` stderr.""" + result = { + 'cycles': 0, + 'cycles_k': 0, + 'cache_misses': 0, + 'stalled_cycles_backend': 0, + 'instructions': 0, + } + for line in text.splitlines(): + m = _PERF_NUM_RE.match(line) + if not m: + continue + raw_val = m.group(1).replace(',', '') + event = m.group(2).strip().rstrip('#').lower() + try: + val = int(float(raw_val)) + except ValueError: + continue + + if event == 'cycles': + result['cycles'] = val + elif event in ('cycles:k', 'cycles:ku'): + result['cycles_k'] = val + elif event in ('cache-misses', 'llc-load-misses'): + result['cache_misses'] = val + elif event in ('stalled-cycles-backend', + 'cpu/stalled-cycles-backend/'): + result['stalled_cycles_backend'] = val + elif event == 'instructions': + result['instructions'] = val + + return result + + +def parse_stress_ng_output(text: str) -> tuple: + """Return (bogo_ops, bogo_ops_per_sec) from stress-ng output.""" + bogo_ops = 0 + bogo_ops_per_sec = 0.0 + + for line in text.splitlines(): + # Modern stress-ng --metrics-brief format: + # stress-ng: info: [PID] cpu N 60.00s ... + # Older format may differ. We look for the "cpu" stressor summary line. + m = re.search( + r'\bcpu\b\s+\d+\s+[\d.]+s\s+([\d.]+)\s+([\d.]+)', line) + if m: + bogo_ops_per_sec = float(m.group(1)) + bogo_ops = int(float(m.group(2))) + break + + # Fallback: "N bogo ops" + m = re.search(r'([\d,]+)\s+bogo ops.*?([\d.]+)\s+bogo ops/s', line) + if m: + bogo_ops = int(m.group(1).replace(',', '')) + bogo_ops_per_sec = float(m.group(2)) + break + + return bogo_ops, bogo_ops_per_sec + + +# --------------------------------------------------------------------------- +# Single-run execution +# --------------------------------------------------------------------------- + +def run_benchmark(config: Config, output_dir: Path) -> BenchmarkResult: + """Execute one benchmark configuration and return a populated result.""" + result = BenchmarkResult(config) + result.output_dir = output_dir + output_dir.mkdir(parents=True, exist_ok=True) + + cg = CgroupManager(config, _next_bench_id()) + sched = SchedulerManager(config) + monitor = CpuUtilMonitor(cg) + + print(f' cgroup: depth={config.depth}, quota={config.quota_str}, ' + f'workers={config.workers}, duration={config.duration}s, ' + f'scheduler={config.scheduler}') + + cg.setup() + _register_cleanup(cg) + sched.setup() + + perf_proc = stress_proc = None + try: + # --- start perf stat (system-wide, runs for exactly duration seconds) --- + perf_cmd = [ + 'perf', 'stat', '-a', + '-e', 'cycles', + '-e', 'cycles:k', + '-e', 'cache-misses', + '-e', 'stalled-cycles-backend', + '-e', 'instructions', + '--', 'sleep', str(config.duration), + ] + perf_proc = subprocess.Popen(perf_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # Give perf a moment to arm before stress-ng starts + time.sleep(SLEEP_PERF_ARM) + + # --- start stress-ng --- + stress_cmd = [ + 'stress-ng', + '--cpu', str(config.workers), + '--timeout', f'{config.duration}s', + '--metrics-brief', + ] + stress_proc = subprocess.Popen(stress_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + # Move stress-ng master and worker children into the leaf cgroup + time.sleep(SLEEP_CGROUP_MOVE) + _move_proc_tree(stress_proc.pid, cg) + + # --- start per-second CPU utilisation monitor --- + monitor.start() + + # --- wait for stress-ng --- + s_out, s_err = stress_proc.communicate(timeout=config.duration + 60) + result.stress_ng_raw = (s_out + s_err).decode('utf-8', errors='replace') + + # --- wait for perf --- + p_out, p_err = perf_proc.communicate(timeout=30) + result.perf_raw = (p_out + p_err).decode('utf-8', errors='replace') + + monitor.stop() + + # --- parse results --- + pm = parse_perf_output(result.perf_raw) + result.cycles = pm['cycles'] + result.cycles_k = pm['cycles_k'] + result.cache_misses = pm['cache_misses'] + result.stalled_cycles_backend = pm['stalled_cycles_backend'] + result.instructions = pm['instructions'] + + result.bogo_ops, result.bogo_ops_per_sec = \ + parse_stress_ng_output(result.stress_ng_raw) + result.cpu_util_samples = list(monitor.samples) + + # --- save raw outputs --- + (output_dir / 'perf_stat.txt').write_text(result.perf_raw) + (output_dir / 'stress_ng.txt').write_text(result.stress_ng_raw) + + except Exception: + monitor.stop() + if perf_proc and perf_proc.poll() is None: + perf_proc.kill() + if stress_proc and stress_proc.poll() is None: + stress_proc.kill() + raise + finally: + sched.teardown() + cg.teardown() + _unregister_cleanup(cg) + + return result + + +def _move_proc_tree(pid: int, cg: CgroupManager): + """Move a process and its direct children into the leaf cgroup.""" + try: + cg.move_pid(pid) + except OSError as exc: + _warn(f'move_pid({pid}): {exc}') + + try: + r = subprocess.run(['pgrep', '-P', str(pid)], + capture_output=True, text=True) + for child in r.stdout.split(): + try: + cg.move_pid(int(child)) + except (OSError, ValueError): + pass + except FileNotFoundError: + pass + + +# --------------------------------------------------------------------------- +# Reporting & graphs +# --------------------------------------------------------------------------- + +def generate_report(groups: dict, output_dir: Path): + """Write report.md (Markdown) and per-group CPU utilisation graphs.""" + lines = [] + lines += [ + '# cpu.max Overhead Benchmark Report', + '', + f'- **Date**: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}', + f'- **CPUs**: {NPROC}', + '', + ] + + for group_key, results in groups.items(): + depth, quota, load_factor = group_key + quota_display = ('max (unlimited)' if quota is None + else f'{quota:g}% ({quota * NPROC / 100:.2f} CPUs)') + + lines += [ + '---', + '', + f'## Group: cgroup_depth={depth} quota={quota_display} load={load_factor}%', + '', + ] + + # Perf metrics table + lines += [ + '| config | cycles:k | cycles | overhead (CPUs) | bogo ops/s |', + '|--------|----------|--------|-----------------|------------|', + ] + for r in results: + lines.append( + f'| `{r.config.name}` ' + f'| {r.cycles_k:,} ' + f'| {r.cycles:,} ' + f'| {r.overhead_cpus:.4f} ' + f'| {r.bogo_ops_per_sec:.1f} |' + ) + lines.append('') + + # Extra perf counters as a detail table (only rows with data) + extra_headers = [] + extra_rows = {r.config.name: {} for r in results} + for r in results: + if r.cache_misses: + extra_headers.append('cache misses') + extra_rows[r.config.name]['cache misses'] = f'{r.cache_misses:,}' + if r.stalled_cycles_backend: + extra_headers.append('stalled cycles (backend)') + extra_rows[r.config.name]['stalled cycles (backend)'] = \ + f'{r.stalled_cycles_backend:,}' + if r.instructions: + extra_headers.append('instructions') + extra_rows[r.config.name]['instructions'] = f'{r.instructions:,}' + + extra_headers = list(dict.fromkeys(extra_headers)) # deduplicate, keep order + if extra_headers: + hdr = '| config | ' + ' | '.join(extra_headers) + ' |' + sep = '|--------|' + '|'.join(['-----'] * len(extra_headers)) + '|' + lines += [hdr, sep] + for r in results: + row_vals = [extra_rows[r.config.name].get(h, '-') + for h in extra_headers] + lines.append(f'| `{r.config.name}` | ' + + ' | '.join(row_vals) + ' |') + lines.append('') + + # Generate graph and embed it + slug = _make_cpu_util_graph(group_key, results, output_dir) + if slug: + lines += [ + '### CPU Utilisation', + '', + f'![CPU utilisation graph]({slug}.png)', + '', + ] + + report_text = '\n'.join(lines) + report_path = output_dir / 'report.md' + report_path.write_text(report_text) + print(f'\nReport written to: {report_path}') + print('\n' + report_text) + + +def _make_cpu_util_graph(group_key: tuple, results: list, output_dir: Path): + """Generate PNG and SVG graphs; return the slug (filename without extension).""" + depth, quota, load_factor = group_key + quota_str = 'max' if quota is None else f'{quota:g}pct' + slug = f'group_d{depth}_{quota_str}_l{load_factor}' + + if not HAS_MATPLOTLIB: + _warn('matplotlib not installed - skipping graphs') + return None + + fig, ax = plt.subplots(figsize=(14, 6)) + colors = plt.cm.tab10.colors + + # Distinct point glyph per scheduler so overlapping lines remain + # distinguishable; runs of the same scheduler are still separated by + # color. + scheduler_markers = { + 'eevdf': 'o', + 'scx_lavd': 's', + } + default_marker = 'D' + + # Samples are stored as CPU-equivalents (1.0 == one CPU fully busy). + # Convert to percent of nproc for the y-axis. + pct_per_cpu = 100.0 / NPROC + + for i, r in enumerate(results): + if not r.cpu_util_samples: + continue + ts = [s[0] for s in r.cpu_util_samples] + vs = [s[1] * pct_per_cpu for s in r.cpu_util_samples] + marker = scheduler_markers.get(r.config.scheduler, default_marker) + ax.plot(ts, vs, label=r.config.name, + color=colors[i % len(colors)], linewidth=1.5, + marker=marker, markersize=5, + markevery=max(1, len(ts) // 30)) + + # Quota reference line (already in percent of nproc). + if quota is not None: + ax.axhline(y=quota, color='red', linestyle='--', linewidth=2, + label=f'quota ({quota:g}%)') + + ax.set_xlabel('Time (s)') + ax.set_ylabel('CPU utilisation (% of nproc)') + quota_title = 'max' if quota is None else f'{quota:g}%' + ax.set_title( + f'CPU utilisation | depth={depth} quota={quota_title} load={load_factor}%') + ax.legend(loc='lower right') + ax.grid(True, alpha=0.3) + ax.set_ylim(bottom=0) + + for fmt in ('png', 'svg'): + path = output_dir / f'{slug}.{fmt}' + fig.savefig(path, format=fmt, dpi=150, bbox_inches='tight') + print(f' graph -> {path}') + + plt.close(fig) + return slug + + +# --------------------------------------------------------------------------- +# Configuration file loader +# --------------------------------------------------------------------------- + +def load_config_file(path: str, select: list = None) -> list: + """ + Parse an INI config file. Each section is one benchmark run. + + Keys (all optional, defaults shown): + cgroup_depth = 32 + quota = max # "max" or a float (percent of nproc) + load_factor = 100 # percent of nproc + duration = 60 # seconds + scheduler = eevdf # eevdf | scx_lavd + scx_path = /usr/bin/scx_lavd + scx_args = --performance --enable-cpu-bw + + @select: optional list of fnmatch globs. If non-empty, only sections + whose name matches at least one pattern are loaded. + """ + cp = configparser.ConfigParser() + if not cp.read(path): + _die(f'cannot read config file: {path}') + + sections = cp.sections() + if select: + sections = [s for s in sections + if any(fnmatch.fnmatchcase(s, p) for p in select)] + if not sections: + available = ', '.join(cp.sections()) or '(none)' + _die(f'no config sections match {select!r}\n' + f'available sections: {available}') + + configs = [] + for section in sections: + quota = _parse_quota(cp.get(section, 'quota', fallback='max')) + configs.append(Config( + name = section, + depth = cp.getint(section, 'cgroup_depth', fallback=32), + quota = quota, + load_factor = cp.getint(section, 'load_factor', fallback=100), + duration = cp.getint(section, 'duration', fallback=60), + scheduler = cp.get(section, 'scheduler', fallback='eevdf'), + scx_path = cp.get(section, 'scx_path', + fallback='/usr/bin/scx_lavd'), + scx_args = cp.get(section, 'scx_args', + fallback='--performance --enable-cpu-bw'), + )) + return configs + + +def _parse_quota(s: str): + """Convert quota string to float (percent of NPROC) or None for "max".""" + s = s.strip() + if s == 'max': + return None + try: + return float(s) + except ValueError: + _die(f'invalid quota value: {s!r} (expected "max" or a number in percent)') + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _fmt_duration(seconds: float) -> str: + """Format a duration in seconds as a human-readable string.""" + seconds = int(seconds) + h, rem = divmod(seconds, 3600) + m, s = divmod(rem, 60) + if h: + return f'{h}h {m:02d}m {s:02d}s' + if m: + return f'{m}m {s:02d}s' + return f'{s}s' + + +def print_estimated_time(configs: list): + """Print per-config and total estimated run time.""" + print('\nEstimated run time:') + total = 0.0 + for cfg in configs: + per_run = ( + cfg.duration + + SLEEP_PERF_ARM + + SLEEP_CGROUP_MOVE + + (SLEEP_SCX_STARTUP if cfg.scheduler == 'scx_lavd' else 0) + ) + total += per_run + scx_note = f' (includes {SLEEP_SCX_STARTUP:.0f}s scx_lavd startup)' \ + if cfg.scheduler == 'scx_lavd' else '' + print(f' [{cfg.name}] {_fmt_duration(per_run)}{scx_note}') + print(f' {"---"}') + print(f' Total {_fmt_duration(total)}') + print() + + +def _warn(msg: str): + print(f'Warning: {msg}', file=sys.stderr) + +def _die(msg: str): + print(f'Error: {msg}', file=sys.stderr) + sys.exit(1) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def build_arg_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + prog='cpu_max_bench.py', + description=( + 'Measure cpu.max cgroup bandwidth overhead using stress-ng + perf stat.\n' + 'Must be run as root.' + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=f"""\ +defaults: depth=32, quota=max, load=100%, duration=60s, scheduler=eevdf + +QUOTA (percent of nproc; this machine has {NPROC} CPUs) + max no cpu.max enforcement (baseline) + 100 nproc CPUs (quota matches the machine) + N N percent of nproc (float accepted, e.g. 12.5 ~= {NPROC * 0.125:g} CPUs) + +CONFIG FILE (INI format) + Each section defines one run; section name becomes the config label. + Example: + + [baseline] + quota = max + scheduler = eevdf + + [cpumax_eevdf] + quota = 100 + scheduler = eevdf + + [cpumax_lavd] + quota = 100 + scheduler = scx_lavd + +EXAMPLES + sudo {sys.argv[0]} + sudo {sys.argv[0]} --quota 100 --scheduler eevdf + sudo {sys.argv[0]} -o /tmp/bench cpu_max_bench.ini + sudo {sys.argv[0]} -o /tmp/bench --scx-path ../target/release/scx_lavd cpu_max_bench.ini + sudo {sys.argv[0]} -o /tmp/bench -S 'baseline_*' cpu_max_bench.ini + sudo {sys.argv[0]} -o /tmp/bench -S 'q50pct_load_010_*' -S 'depth_01_*' cpu_max_bench.ini +""") + + p.add_argument('config_file', nargs='?', + help='INI file with multiple benchmark configurations') + p.add_argument('-S', '--select', metavar='PATTERN', action='append', + default=[], + help='only run config sections whose name matches PATTERN ' + '(fnmatch glob); may be repeated to select the union ' + '(e.g. -S "baseline_*" -S q50pct_load_010_eevdf)') + p.add_argument('-d', '--depth', type=int, default=32, metavar='N', + help='cgroup hierarchy depth (default: 32)') + p.add_argument('-q', '--quota', default='max', metavar='QUOTA', + help='CPU quota: max | (default: max)') + p.add_argument('-l', '--load-factor', type=int, default=100, metavar='PCT', + help='stress-ng workers as %% of nproc (default: 100)') + p.add_argument('-t', '--duration', type=int, default=60, metavar='SECS', + help='benchmark duration in seconds (default: 60)') + p.add_argument('-s', '--scheduler', choices=['eevdf', 'scx_lavd'], + default='eevdf', + help='CPU scheduler to use (default: eevdf)') + p.add_argument('--scx-path', default=None, metavar='PATH', + help='path to scx_lavd binary (default: /usr/bin/scx_lavd); ' + 'overrides scx_path in the config file for all configurations') + p.add_argument('--scx-args', + default='--performance --enable-cpu-bw', metavar='ARGS', + help='arguments for scx_lavd ' + '(default: --performance --enable-cpu-bw)') + p.add_argument('-o', '--output', default=None, metavar='DIR', + help='output directory (default: YYYY-MM-DDTHH:MM:SS)') + return p + + +_DISTROS = [ + # (display name, install command prefix, package-key in _TOOLS / _PY_PKGS) + ('Ubuntu', 'sudo apt install', 'ubuntu'), + ('Arch Linux', 'sudo pacman -S', 'arch'), + ('Fedora or Amazon Linux', 'sudo dnf install', 'fedora'), # AL2023; on AL2 swap dnf -> yum +] + +_TOOLS = { + 'perf': { + 'binary': 'perf', + 'extra_args': None, + 'ubuntu': 'linux-tools-common linux-tools-generic', + 'arch': 'perf', + 'fedora': 'perf', + }, + 'perf stat': { + 'binary': 'perf', + 'extra_args': ['stat', '--help'], + 'ubuntu': 'linux-tools-common linux-tools-generic', + 'arch': 'perf', + 'fedora': 'perf', + }, + 'stress-ng': { + 'binary': 'stress-ng', + 'extra_args': None, + 'ubuntu': 'stress-ng', + 'arch': 'stress-ng', + 'fedora': 'stress-ng', + }, +} + +_PY_PKGS = { + 'matplotlib': { + 'ubuntu': 'python3-matplotlib', + 'arch': 'python-matplotlib', + 'fedora': 'python3-matplotlib', + }, +} + + +def check_dependencies(): + """Verify required tools and Python packages are available.""" + missing_tools = [] + for name, info in _TOOLS.items(): + # First check the binary is on PATH + r = subprocess.run(['which', info['binary']], capture_output=True) + if r.returncode != 0: + missing_tools.append(name) + continue + # For "perf stat", also verify the subcommand is available + if info['extra_args']: + r2 = subprocess.run([info['binary']] + info['extra_args'], + capture_output=True) + if r2.returncode != 0: + missing_tools.append(name) + + missing_py = [] + if not HAS_MATPLOTLIB: + missing_py.append('matplotlib') + + if not missing_tools and not missing_py: + return + + print('ERROR: missing required dependencies\n', file=sys.stderr) + + if missing_tools: + print('Missing tools:', ', '.join(missing_tools), file=sys.stderr) + for display, cmd, key in _DISTROS: + pkgs = list(dict.fromkeys( + _TOOLS[t][key] for t in missing_tools if t in _TOOLS)) + print(f'\nInstall on {display}:', file=sys.stderr) + print(f' {cmd} {" ".join(pkgs)}', file=sys.stderr) + + if missing_py: + print('\nMissing Python packages:', ', '.join(missing_py), + file=sys.stderr) + for display, cmd, key in _DISTROS: + pkgs = list(dict.fromkeys( + _PY_PKGS[p][key] for p in missing_py if p in _PY_PKGS)) + print(f'\nInstall on {display}:', file=sys.stderr) + print(f' {cmd} {" ".join(pkgs)}', file=sys.stderr) + print('\nOr via pip:', file=sys.stderr) + print(f' pip3 install {" ".join(missing_py)}', file=sys.stderr) + + sys.exit(1) + + +def main(): + parser = build_arg_parser() + args = parser.parse_args() + + # Install handlers so a SIGTERM / SIGINT mid-run still tears down + # any cgroups currently set up. atexit covers normal/exception + # exits; the signal handler covers external termination. + signal.signal(signal.SIGTERM, _signal_handler) + signal.signal(signal.SIGINT, _signal_handler) + + check_dependencies() + + if os.geteuid() != 0: + _die('must be run as root (needs cgroup writes and perf_event access)') + + # Resolve output directory + out_root = Path(args.output) if args.output else \ + Path(datetime.now().strftime('%Y-%m-%dT%H:%M:%S')) + out_root.mkdir(parents=True, exist_ok=True) + print(f'Output directory: {out_root}') + + # Build configuration list + if args.config_file: + configs = load_config_file(args.config_file, select=args.select) + if not configs: + _die(f'no configurations found in {args.config_file}') + if args.scx_path is not None: + for cfg in configs: + cfg.scx_path = args.scx_path + elif args.select: + _die('--select can only be used with a config file') + else: + configs = [Config( + name = 'benchmark', + depth = args.depth, + quota = _parse_quota(args.quota), + load_factor = args.load_factor, + duration = args.duration, + scheduler = args.scheduler, + scx_path = args.scx_path or '/usr/bin/scx_lavd', + scx_args = args.scx_args, + )] + + print_estimated_time(configs) + + # Run benchmarks + all_results = [] + for idx, cfg in enumerate(configs, 1): + print(f'\n[{idx}/{len(configs)}] {cfg.name}') + cfg_dir = out_root / f'config_{cfg.name}' + result = run_benchmark(cfg, cfg_dir) + all_results.append(result) + print(f' cycles:k / cycles = {result.cycles_k:,} / {result.cycles:,}') + print(f' overhead = {result.overhead_cpus:.4f} CPUs') + print(f' bogo ops/s = {result.bogo_ops_per_sec:.1f}') + + # Group and report + groups: dict = {} + for r in all_results: + groups.setdefault(r.config.group_key, []).append(r) + + print(f'\n{"=" * 80}') + print('Generating report...') + generate_report(groups, out_root) + print(f'\nDone. Results in: {out_root}/ (report: {out_root}/report.md)') + + +if __name__ == '__main__': + main()