Skip to content

RFC: scx_utils, scx/compat: Recover enum64 values truncated by kernel BTF #3708

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ByteLab-David wants to merge 1 commit into
base: main
Choose a base branch
from
Open

RFC: scx_utils, scx/compat: Recover enum64 values truncated by kernel BTF #3708

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 139 additions & 4 deletions rust/scx_utils/src/compat.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ const PROCFS_MOUNTS: &str = "/proc/mounts";
const TRACEFS: &str = "tracefs";
const DEBUGFS: &str = "debugfs";

mod enums_abi {
include!("enums_abi.autogen.rs");
}

lazy_static::lazy_static! {
pub static ref SCX_OPS_KEEP_BUILTIN_IDLE: u64 =
read_enum("scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE").unwrap_or(0);
Expand Down Expand Up @@ -103,11 +107,71 @@ fn btf_name_str_by_offset(btf: &btf, name_off: u32) -> Result<&str> {
.with_context(|| format!("Failed to convert {:?} to string", n))?)
}

/// Recover the true value of a 64-bit enum enumerator whose kernel BTF entry
/// was truncated to its low 32 bits.
///
/// Kernels whose BTF was generated without BTF_KIND_ENUM64 support encode
/// 64-bit enums as 8-byte BTF_KIND_ENUM entries whose enumerator values only
/// carry the low 32 bits. This happens with pahole < 1.24, which predates
/// ENUM64 (e.g. Google's Container-Optimized OS builds with pahole 1.23),
/// and with newer pahole passing --skip_encoding_btf_enum64. The high bits
/// can't be recovered from kernel BTF, so substitute the value from the
/// vmlinux.h this tree was built against, cross-checked against the low 32
/// bits the kernel did provide.
fn recover_truncated_enum64_from(
table: &[(&str, &str, u64)],
type_name: &str,
name: &str,
lo32: u32,
) -> Result<u64> {
static WARN_ONCE: std::sync::Once = std::sync::Once::new();

let Some(&(_, _, abi_val)) = table.iter().find(|(t, n, _)| *t == type_name && *n == name)
else {
// Unknown enumerator (likely a stale autogen table). Fail
// pessimistically to avoid returning an invalid value.
bail!(
"kernel BTF truncates 64-bit enum {}::{} to 0x{:x}; 64-bit \
variant not found in vmlinux.h",
type_name,
name,
lo32
);
};

if abi_val <= u32::MAX as u64 {
return Ok(lo32 as u64);
}

if abi_val as u32 != lo32 {
bail!(
"kernel BTF value of {}::{} (0x{:x}) doesn't match the low 32 bits \
of the vmlinux.h value (0x{:x}); refusing to substitute",
type_name,
name,
lo32,
abi_val
);
}

WARN_ONCE.call_once(|| {
warn!(
"kernel BTF lacks ENUM64 encoding; substituting vmlinux.h values \
for truncated 64-bit scx enums"
);
});
Ok(abi_val)
}

fn recover_truncated_enum64(type_name: &str, name: &str, lo32: u32) -> Result<u64> {
recover_truncated_enum64_from(enums_abi::ENUM_ABI_VALUES, type_name, name, lo32)
}

pub fn read_enum(type_name: &str, name: &str) -> Result<u64> {
let btf: &btf = *VMLINUX_BTF;

let type_name = CString::new(type_name).unwrap();
let tid = unsafe { btf__find_by_name(btf, type_name.as_ptr()) };
let c_type_name = CString::new(type_name).unwrap();
let tid = unsafe { btf__find_by_name(btf, c_type_name.as_ptr()) };
if tid < 0 {
bail!("type {:?} doesn't exist, ret={}", type_name, tid);
}
Expand All @@ -122,6 +186,11 @@ pub fn read_enum(type_name: &str, name: &str) -> Result<u64> {
BTF_KIND_ENUM => {
for e in btf_enum(t).iter() {
if btf_name_str_by_offset(btf, e.name_off)? == name {
// Try to recover a 64-bit enum from an 8-byte BTF_KIND_ENUM that was encoded
// with an insufficiently modern pahole.
if unsafe { t.__bindgen_anon_1.size } == 8 {
return recover_truncated_enum64(type_name, name, e.val as u32);
}
return Ok(e.val as u64);
}
}
Expand Down Expand Up @@ -161,8 +230,8 @@ pub fn read_enum_any(type_names: &[&str], name: &str) -> Result<u64> {
pub fn struct_has_field(type_name: &str, field: &str) -> Result<bool> {
let btf: &btf = *VMLINUX_BTF;

let type_name = CString::new(type_name).unwrap();
let tid = unsafe { btf__find_by_name_kind(btf, type_name.as_ptr(), BTF_KIND_STRUCT) };
let c_type_name = CString::new(type_name).unwrap();
let tid = unsafe { btf__find_by_name_kind(btf, c_type_name.as_ptr(), BTF_KIND_STRUCT) };
if tid < 0 {
bail!("type {:?} doesn't exist, ret={}", type_name, tid);
}
Expand Down Expand Up @@ -652,6 +721,72 @@ mod tests {
assert!(super::read_enum_any(&["NO_SUCH_TYPE", "pid_type"], "NO_SUCH_ENUM").is_err());
}

#[test]
fn test_recover_truncated_enum64() {
let table: &[(&str, &str, u64)] = &[
("scx_dsq_id_flags", "SCX_DSQ_LOCAL", 0x8000000000000002),
("scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000),
("scx_enq_flags", "SCX_ENQ_HEAD", 0x10000),
];

// >32-bit values with matching low bits get substituted.
assert_eq!(
super::recover_truncated_enum64_from(table, "scx_dsq_id_flags", "SCX_DSQ_LOCAL", 2)
.unwrap(),
0x8000000000000002
);
assert_eq!(
super::recover_truncated_enum64_from(table, "scx_enq_flags", "SCX_ENQ_PREEMPT", 0)
.unwrap(),
0x100000000
);
// Sub-32-bit values truncate losslessly, so the kernel's value stays
// authoritative even when it disagrees with the table.
assert_eq!(
super::recover_truncated_enum64_from(table, "scx_enq_flags", "SCX_ENQ_HEAD", 0x20000)
.unwrap(),
0x20000
);
// A low-32 mismatch on a >32-bit value is ABI drift; refuse.
assert!(super::recover_truncated_enum64_from(
table,
"scx_dsq_id_flags",
"SCX_DSQ_LOCAL",
3
)
.is_err());
// Unknown enumerators fail pessimistically (stale autogen table).
assert!(
super::recover_truncated_enum64_from(table, "scx_enq_flags", "SCX_ENQ_NEW", 7).is_err()
);
}

#[test]
fn test_enum_abi_table() {
// Spot-check the autogenerated table against ABI values that have
// been stable on every kernel that ships sched_ext.
let find = |t: &str, n: &str| {
super::enums_abi::ENUM_ABI_VALUES
.iter()
.find(|(ty, na, _)| *ty == t && *na == n)
.map(|&(_, _, v)| v)
};
assert_eq!(
find("scx_dsq_id_flags", "SCX_DSQ_FLAG_BUILTIN"),
Some(1 << 63)
);
assert_eq!(
find("scx_dsq_id_flags", "SCX_DSQ_LOCAL"),
Some((1 << 63) | 2)
);
assert_eq!(
find("scx_dsq_id_flags", "SCX_DSQ_LOCAL_ON"),
Some((1 << 63) | (1 << 62))
);
assert_eq!(find("scx_public_consts", "SCX_SLICE_INF"), Some(u64::MAX));
assert_eq!(find("scx_enq_flags", "SCX_ENQ_PREEMPT"), Some(1 << 32));
}

#[test]
fn test_struct_has_field() {
assert!(super::struct_has_field("task_struct", "flags").unwrap());
Expand Down
158 changes: 158 additions & 0 deletions rust/scx_utils/src/enums_abi.autogen.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
// WARNING: This file is autogenerated from gen_enum_defs.py [1].
//
// scx enumerator values from the vmlinux.h this tree is built against.
// Used as the substitution source when the running kernel's BTF lacks
// BTF_KIND_ENUM64 encoding and 64-bit enum values are truncated.
//
// [1] https://github.com/sched-ext/scx/blob/main/scripts/gen_enum_defs.py

/// (enum type name, enumerator name, value) triples from vmlinux.h.
pub const ENUM_ABI_VALUES: &[(&str, &str, u64)] = &[
("scx_consts", "SCX_DSP_DFL_MAX_BATCH", 0x20),
("scx_consts", "SCX_DSP_MAX_LOOPS", 0x20),
("scx_consts", "SCX_WATCHDOG_MAX_TIMEOUT", 0x7530),
("scx_consts", "SCX_EXIT_BT_LEN", 0x40),
("scx_consts", "SCX_EXIT_MSG_LEN", 0x400),
("scx_consts", "SCX_EXIT_DUMP_DFL_LEN", 0x8000),
("scx_consts", "SCX_CPUPERF_ONE", 0x400),
("scx_consts", "SCX_TASK_ITER_BATCH", 0x20),
("scx_consts", "SCX_BYPASS_HOST_NTH", 0x2),
("scx_consts", "SCX_BYPASS_LB_DFL_INTV_US", 0x7a120),
("scx_consts", "SCX_BYPASS_LB_DONOR_PCT", 0x7d),
("scx_consts", "SCX_BYPASS_LB_MIN_DELTA_DIV", 0x4),
("scx_consts", "SCX_BYPASS_LB_BATCH", 0x100),
("scx_consts", "SCX_REENQ_LOCAL_MAX_REPEAT", 0x100),
("scx_consts", "SCX_SUB_MAX_DEPTH", 0x4),
("scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_RT", 0x0),
("scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_DL", 0x1),
("scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_STOP", 0x2),
("scx_cpu_preempt_reason", "SCX_CPU_PREEMPT_UNKNOWN", 0x3),
("scx_deq_flags", "SCX_DEQ_SLEEP", 0x1),
("scx_deq_flags", "SCX_DEQ_CORE_SCHED_EXEC", 0x100000000),
("scx_deq_flags", "SCX_DEQ_SCHED_CHANGE", 0x200000000),
("scx_dsq_id_flags", "SCX_DSQ_FLAG_BUILTIN", 0x8000000000000000),
("scx_dsq_id_flags", "SCX_DSQ_FLAG_LOCAL_ON", 0x4000000000000000),
("scx_dsq_id_flags", "SCX_DSQ_INVALID", 0x8000000000000000),
("scx_dsq_id_flags", "SCX_DSQ_GLOBAL", 0x8000000000000001),
("scx_dsq_id_flags", "SCX_DSQ_LOCAL", 0x8000000000000002),
("scx_dsq_id_flags", "SCX_DSQ_BYPASS", 0x8000000000000003),
("scx_dsq_id_flags", "SCX_DSQ_LOCAL_ON", 0xc000000000000000),
("scx_dsq_id_flags", "SCX_DSQ_LOCAL_CPU_MASK", 0xffffffff),
("scx_dsq_iter_flags", "SCX_DSQ_ITER_REV", 0x10000),
("scx_dsq_iter_flags", "__SCX_DSQ_ITER_HAS_SLICE", 0x40000000),
("scx_dsq_iter_flags", "__SCX_DSQ_ITER_HAS_VTIME", 0x80000000),
("scx_dsq_iter_flags", "__SCX_DSQ_ITER_USER_FLAGS", 0x10000),
("scx_dsq_iter_flags", "__SCX_DSQ_ITER_ALL_FLAGS", 0xc0010000),
("scx_dsq_lnode_flags", "SCX_DSQ_LNODE_ITER_CURSOR", 0x1),
("scx_dsq_lnode_flags", "__SCX_DSQ_LNODE_PRIV_SHIFT", 0x10),
("scx_enable_state", "SCX_ENABLING", 0x0),
("scx_enable_state", "SCX_ENABLED", 0x1),
("scx_enable_state", "SCX_DISABLING", 0x2),
("scx_enable_state", "SCX_DISABLED", 0x3),
("scx_enq_flags", "SCX_ENQ_WAKEUP", 0x1),
("scx_enq_flags", "SCX_ENQ_HEAD", 0x10000),
("scx_enq_flags", "SCX_ENQ_CPU_SELECTED", 0x100000),
("scx_enq_flags", "SCX_ENQ_PREEMPT", 0x100000000),
("scx_enq_flags", "SCX_ENQ_IMMED", 0x200000000),
("scx_enq_flags", "SCX_ENQ_REENQ", 0x10000000000),
("scx_enq_flags", "SCX_ENQ_LAST", 0x20000000000),
("scx_enq_flags", "__SCX_ENQ_INTERNAL_MASK", 0xff00000000000000),
("scx_enq_flags", "SCX_ENQ_CLEAR_OPSS", 0x100000000000000),
("scx_enq_flags", "SCX_ENQ_DSQ_PRIQ", 0x200000000000000),
("scx_enq_flags", "SCX_ENQ_NESTED", 0x400000000000000),
("scx_enq_flags", "SCX_ENQ_GDSQ_FALLBACK", 0x800000000000000),
("scx_ent_dsq_flags", "SCX_TASK_DSQ_ON_PRIQ", 0x1),
("scx_ent_flags", "SCX_TASK_QUEUED", 0x1),
("scx_ent_flags", "SCX_TASK_IN_CUSTODY", 0x2),
("scx_ent_flags", "SCX_TASK_RESET_RUNNABLE_AT", 0x4),
("scx_ent_flags", "SCX_TASK_DEQD_FOR_SLEEP", 0x8),
("scx_ent_flags", "SCX_TASK_SUB_INIT", 0x10),
("scx_ent_flags", "SCX_TASK_IMMED", 0x20),
("scx_ent_flags", "SCX_TASK_STATE_SHIFT", 0x8),
("scx_ent_flags", "SCX_TASK_STATE_BITS", 0x2),
("scx_ent_flags", "SCX_TASK_STATE_MASK", 0x300),
("scx_ent_flags", "SCX_TASK_NONE", 0x0),
("scx_ent_flags", "SCX_TASK_INIT", 0x100),
("scx_ent_flags", "SCX_TASK_READY", 0x200),
("scx_ent_flags", "SCX_TASK_ENABLED", 0x300),
("scx_ent_flags", "SCX_TASK_REENQ_REASON_SHIFT", 0xc),
("scx_ent_flags", "SCX_TASK_REENQ_REASON_BITS", 0x2),
("scx_ent_flags", "SCX_TASK_REENQ_REASON_MASK", 0x3000),
("scx_ent_flags", "SCX_TASK_REENQ_NONE", 0x0),
("scx_ent_flags", "SCX_TASK_REENQ_KFUNC", 0x1000),
("scx_ent_flags", "SCX_TASK_REENQ_IMMED", 0x2000),
("scx_ent_flags", "SCX_TASK_REENQ_PREEMPTED", 0x3000),
("scx_ent_flags", "SCX_TASK_CURSOR", 0xffffffff80000000),
("scx_exit_code", "SCX_ECODE_RSN_HOTPLUG", 0x100000000),
("scx_exit_code", "SCX_ECODE_RSN_CGROUP_OFFLINE", 0x200000000),
("scx_exit_code", "SCX_ECODE_ACT_RESTART", 0x1000000000000),
("scx_exit_flags", "SCX_EFLAG_INITIALIZED", 0x1),
("scx_exit_kind", "SCX_EXIT_NONE", 0x0),
("scx_exit_kind", "SCX_EXIT_DONE", 0x1),
("scx_exit_kind", "SCX_EXIT_UNREG", 0x40),
("scx_exit_kind", "SCX_EXIT_UNREG_BPF", 0x41),
("scx_exit_kind", "SCX_EXIT_UNREG_KERN", 0x42),
("scx_exit_kind", "SCX_EXIT_SYSRQ", 0x43),
("scx_exit_kind", "SCX_EXIT_PARENT", 0x44),
("scx_exit_kind", "SCX_EXIT_ERROR", 0x400),
("scx_exit_kind", "SCX_EXIT_ERROR_BPF", 0x401),
("scx_exit_kind", "SCX_EXIT_ERROR_STALL", 0x402),
("scx_kf_mask", "SCX_KF_UNLOCKED", 0x0),
("scx_kf_mask", "SCX_KF_CPU_RELEASE", 0x1),
("scx_kf_mask", "SCX_KF_DISPATCH", 0x2),
("scx_kf_mask", "SCX_KF_ENQUEUE", 0x4),
("scx_kf_mask", "SCX_KF_SELECT_CPU", 0x8),
("scx_kf_mask", "SCX_KF_REST", 0x10),
("scx_kf_mask", "__SCX_KF_RQ_LOCKED", 0x1f),
("scx_kf_mask", "__SCX_KF_TERMINAL", 0x1c),
("scx_kick_flags", "SCX_KICK_IDLE", 0x1),
("scx_kick_flags", "SCX_KICK_PREEMPT", 0x2),
("scx_kick_flags", "SCX_KICK_WAIT", 0x4),
("scx_opi", "SCX_OPI_BEGIN", 0x0),
("scx_opi", "SCX_OPI_NORMAL_BEGIN", 0x0),
("scx_opi", "SCX_OPI_NORMAL_END", 0x21),
("scx_opi", "SCX_OPI_CPU_HOTPLUG_BEGIN", 0x21),
("scx_opi", "SCX_OPI_CPU_HOTPLUG_END", 0x23),
("scx_opi", "SCX_OPI_END", 0x23),
("scx_ops_flags", "SCX_OPS_KEEP_BUILTIN_IDLE", 0x1),
("scx_ops_flags", "SCX_OPS_ENQ_LAST", 0x2),
("scx_ops_flags", "SCX_OPS_ENQ_EXITING", 0x4),
("scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL", 0x8),
("scx_ops_flags", "SCX_OPS_ENQ_MIGRATION_DISABLED", 0x10),
("scx_ops_flags", "SCX_OPS_ALLOW_QUEUED_WAKEUP", 0x20),
("scx_ops_flags", "SCX_OPS_BUILTIN_IDLE_PER_NODE", 0x40),
("scx_ops_flags", "SCX_OPS_ALWAYS_ENQ_IMMED", 0x80),
("scx_ops_flags", "SCX_OPS_ALL_FLAGS", 0xff),
("scx_ops_flags", "__SCX_OPS_INTERNAL_MASK", 0xff00000000000000),
("scx_ops_flags", "SCX_OPS_HAS_CPU_PREEMPT", 0x100000000000000),
("scx_ops_state", "SCX_OPSS_NONE", 0x0),
("scx_ops_state", "SCX_OPSS_QUEUEING", 0x1),
("scx_ops_state", "SCX_OPSS_QUEUED", 0x2),
("scx_ops_state", "SCX_OPSS_DISPATCHING", 0x3),
("scx_ops_state", "SCX_OPSS_QSEQ_SHIFT", 0x2),
("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_CORE", 0x1),
("scx_pick_idle_cpu_flags", "SCX_PICK_IDLE_IN_NODE", 0x2),
("scx_public_consts", "SCX_OPS_NAME_LEN", 0x80),
("scx_public_consts", "SCX_SLICE_DFL", 0x1312d00),
("scx_public_consts", "SCX_SLICE_BYPASS", 0x4c4b40),
("scx_public_consts", "SCX_SLICE_INF", 0xffffffffffffffff),
("scx_reenq_flags", "SCX_REENQ_ANY", 0x1),
("scx_reenq_flags", "__SCX_REENQ_FILTER_MASK", 0xffff),
("scx_reenq_flags", "__SCX_REENQ_USER_MASK", 0x1),
("scx_reenq_flags", "SCX_REENQ_TSR_RQ_OPEN", 0x100000000),
("scx_reenq_flags", "SCX_REENQ_TSR_NOT_FIRST", 0x200000000),
("scx_reenq_flags", "__SCX_REENQ_TSR_MASK", 0xf00000000),
("scx_rq_flags", "SCX_RQ_ONLINE", 0x1),
("scx_rq_flags", "SCX_RQ_CAN_STOP_TICK", 0x2),
("scx_rq_flags", "SCX_RQ_BAL_KEEP", 0x8),
("scx_rq_flags", "SCX_RQ_CLK_VALID", 0x20),
("scx_rq_flags", "SCX_RQ_BAL_CB_PENDING", 0x40),
("scx_rq_flags", "SCX_RQ_IN_WAKEUP", 0x10000),
("scx_rq_flags", "SCX_RQ_IN_BALANCE", 0x20000),
("scx_sched_pcpu_flags", "SCX_SCHED_PCPU_BYPASSING", 0x1),
("scx_tg_flags", "SCX_TG_ONLINE", 0x1),
("scx_tg_flags", "SCX_TG_INITED", 0x2),
("scx_wake_flags", "SCX_WAKE_FORK", 0x4),
("scx_wake_flags", "SCX_WAKE_TTWU", 0x8),
("scx_wake_flags", "SCX_WAKE_SYNC", 0x10),
];
Loading
Loading