diff --git a/scheds/rust/scx_layered/src/bpf/intf.h b/scheds/rust/scx_layered/src/bpf/intf.h index 29fe4ea854..80838622ad 100644 --- a/scheds/rust/scx_layered/src/bpf/intf.h +++ b/scheds/rust/scx_layered/src/bpf/intf.h @@ -46,6 +46,7 @@ enum consts { LAYER_LAT_DECAY_FACTOR = 32, CLEAR_PREEMPTING_AFTER = 10000000, /* 10ms */ NUM_PROXIMITY_MAPS = 32, + NR_RUNQ_LAT_BUCKETS = 26, DSQ_ID_SPECIAL_MASK = 0xc0000000, HI_FB_DSQ_BASE = 0x40000000, @@ -156,6 +157,8 @@ enum layer_stat_id { LSTAT_LLC_DRAIN_TRY, LSTAT_LLC_DRAIN, LSTAT_SKIP_REMOTE_NODE, + LSTAT_RUNQ_LAT_BASE, + LSTAT_RUNQ_LAT_END = LSTAT_RUNQ_LAT_BASE + NR_RUNQ_LAT_BUCKETS - 1, NR_LSTATS, }; diff --git a/scheds/rust/scx_layered/src/bpf/main.bpf.c b/scheds/rust/scx_layered/src/bpf/main.bpf.c index 8da42b8317..3e29cdd10f 100644 --- a/scheds/rust/scx_layered/src/bpf/main.bpf.c +++ b/scheds/rust/scx_layered/src/bpf/main.bpf.c @@ -711,6 +711,8 @@ struct task_ctx { u64 layer_refresh_seq; u64 recheck_layer_membership; + + bool runq_lat_pending; }; struct { @@ -3522,6 +3524,7 @@ void BPF_STRUCT_OPS(layered_runnable, struct task_struct *p, u64 enq_flags) return; taskc->runnable_at = now; + taskc->runq_lat_pending = true; maybe_refresh_layer(p, taskc, now); if (enq_flags & SCX_ENQ_WAKEUP) @@ -3601,6 +3604,34 @@ void BPF_STRUCT_OPS(layered_running, struct task_struct *p) cpuc->running_open = false; } + if (taskc->runq_lat_pending) { + u64 lat_us = (now - taskc->runnable_at) / 1000; + u32 rql_bucket = 0; + + taskc->runq_lat_pending = false; + if (lat_us >> 16) { + rql_bucket += 16; + lat_us >>= 16; + } + if (lat_us >> 8) { + rql_bucket += 8; + lat_us >>= 8; + } + if (lat_us >> 4) { + rql_bucket += 4; + lat_us >>= 4; + } + if (lat_us >> 2) { + rql_bucket += 2; + lat_us >>= 2; + } + if (lat_us >> 1) + rql_bucket += 1; + if (rql_bucket >= NR_RUNQ_LAT_BUCKETS) + rql_bucket = NR_RUNQ_LAT_BUCKETS - 1; + lstat_inc(LSTAT_RUNQ_LAT_BASE + rql_bucket, layer, cpuc); + } + /* * If this CPU is transitioning from running an exclusive task to a * non-exclusive one, the sibling CPU has likely been idle. Wake it up. diff --git a/scheds/rust/scx_layered/src/stats.rs b/scheds/rust/scx_layered/src/stats.rs index e2508ac0b6..c299a96e8b 100644 --- a/scheds/rust/scx_layered/src/stats.rs +++ b/scheds/rust/scx_layered/src/stats.rs @@ -78,6 +78,9 @@ const LSTAT_LLC_DRAIN_TRY: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN_TRY a const LSTAT_LLC_DRAIN: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN as usize; const LSTAT_SKIP_REMOTE_NODE: usize = bpf_intf::layer_stat_id_LSTAT_SKIP_REMOTE_NODE as usize; +const LSTAT_RUNQ_LAT_BASE: usize = bpf_intf::layer_stat_id_LSTAT_RUNQ_LAT_BASE as usize; +const NR_RUNQ_LAT_BUCKETS: usize = bpf_intf::consts_NR_RUNQ_LAT_BUCKETS as usize; + const LLC_LSTAT_LAT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_LAT as usize; const LLC_LSTAT_CNT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_CNT as usize; @@ -251,6 +254,8 @@ pub struct LayerStats { pub node_loads: Vec, #[stat(desc = "Whether xnuma gating is active for this layer (0/1)")] pub xnuma_active: u32, + #[stat(desc = "runqueue latency histogram, log2 us buckets 1us..32s, per stats interval")] + pub l_runq_lat_hist: Vec, } impl LayerStats { @@ -390,6 +395,9 @@ impl LayerStats { .map(|l| l * 100.0) .collect(), xnuma_active: if xnuma_active { 1 } else { 0 }, + l_runq_lat_hist: (0..NR_RUNQ_LAT_BUCKETS) + .map(|b| lstat(LSTAT_RUNQ_LAT_BASE + b)) + .collect(), } }