scx_layered: per-layer log2 runqueue-latency histogram (runnable -> running)

scheds/rust/scx_layered/src/bpf/intf.h

@@ -46,6 +46,7 @@ enum consts {
 	LAYER_LAT_DECAY_FACTOR	= 32,
 	CLEAR_PREEMPTING_AFTER	= 10000000,	/* 10ms */
 	NUM_PROXIMITY_MAPS = 32,
+	NR_RUNQ_LAT_BUCKETS	= 26,
 
 	DSQ_ID_SPECIAL_MASK	= 0xc0000000,
 	HI_FB_DSQ_BASE		= 0x40000000,
@@ -156,6 +157,8 @@ enum layer_stat_id {
 	LSTAT_LLC_DRAIN_TRY,
 	LSTAT_LLC_DRAIN,
 	LSTAT_SKIP_REMOTE_NODE,
+	LSTAT_RUNQ_LAT_BASE,
+	LSTAT_RUNQ_LAT_END = LSTAT_RUNQ_LAT_BASE + NR_RUNQ_LAT_BUCKETS - 1,
 	NR_LSTATS,
 };
 

scheds/rust/scx_layered/src/bpf/main.bpf.c

@@ -711,6 +711,8 @@ struct task_ctx {
 	u64			layer_refresh_seq;
 
 	u64			recheck_layer_membership;
+
+	bool			runq_lat_pending;
 };
 
 struct {
@@ -3522,6 +3524,7 @@ void BPF_STRUCT_OPS(layered_runnable, struct task_struct *p, u64 enq_flags)
 		return;
 
 	taskc->runnable_at = now;
+	taskc->runq_lat_pending = true;
 	maybe_refresh_layer(p, taskc, now);
 
 	if (enq_flags & SCX_ENQ_WAKEUP)
@@ -3601,6 +3604,34 @@ void BPF_STRUCT_OPS(layered_running, struct task_struct *p)
 		cpuc->running_open = false;
 	}
 
+	if (taskc->runq_lat_pending) {
+		u64 lat_us = (now - taskc->runnable_at) / 1000;
+		u32 rql_bucket = 0;
+
+		taskc->runq_lat_pending = false;
+		if (lat_us >> 16) {
+			rql_bucket += 16;
+			lat_us >>= 16;
+		}
+		if (lat_us >> 8) {
+			rql_bucket += 8;
+			lat_us >>= 8;
+		}
+		if (lat_us >> 4) {
+			rql_bucket += 4;
+			lat_us >>= 4;
+		}
+		if (lat_us >> 2) {
+			rql_bucket += 2;
+			lat_us >>= 2;
+		}
+		if (lat_us >> 1)
+			rql_bucket += 1;
+		if (rql_bucket >= NR_RUNQ_LAT_BUCKETS)
+			rql_bucket = NR_RUNQ_LAT_BUCKETS - 1;
+		lstat_inc(LSTAT_RUNQ_LAT_BASE + rql_bucket, layer, cpuc);
+	}
+
 	/*
 	 * If this CPU is transitioning from running an exclusive task to a
 	 * non-exclusive one, the sibling CPU has likely been idle. Wake it up.

scheds/rust/scx_layered/src/stats.rs

@@ -78,6 +78,9 @@ const LSTAT_LLC_DRAIN_TRY: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN_TRY a
 const LSTAT_LLC_DRAIN: usize = bpf_intf::layer_stat_id_LSTAT_LLC_DRAIN as usize;
 const LSTAT_SKIP_REMOTE_NODE: usize = bpf_intf::layer_stat_id_LSTAT_SKIP_REMOTE_NODE as usize;
 
+const LSTAT_RUNQ_LAT_BASE: usize = bpf_intf::layer_stat_id_LSTAT_RUNQ_LAT_BASE as usize;
+const NR_RUNQ_LAT_BUCKETS: usize = bpf_intf::consts_NR_RUNQ_LAT_BUCKETS as usize;
+
 const LLC_LSTAT_LAT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_LAT as usize;
 const LLC_LSTAT_CNT: usize = bpf_intf::llc_layer_stat_id_LLC_LSTAT_CNT as usize;
 
@@ -251,6 +254,8 @@ pub struct LayerStats {
     pub node_loads: Vec<f64>,
     #[stat(desc = "Whether xnuma gating is active for this layer (0/1)")]
     pub xnuma_active: u32,
+    #[stat(desc = "runqueue latency histogram, log2 us buckets 1us..32s, per stats interval")]
+    pub l_runq_lat_hist: Vec<u64>,
 }
 
 impl LayerStats {
@@ -390,6 +395,9 @@ impl LayerStats {
                 .map(|l| l * 100.0)
                 .collect(),
             xnuma_active: if xnuma_active { 1 } else { 0 },
+            l_runq_lat_hist: (0..NR_RUNQ_LAT_BUCKETS)
+                .map(|b| lstat(LSTAT_RUNQ_LAT_BASE + b))
+                .collect(),
         }
     }