From 6401a21ba8d3051424dc7a5ac568d380a4f1635a Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Thu, 30 Jul 2026 13:39:38 -0400 Subject: [PATCH] Let systemd-oomd kill a runaway app instead of losing the session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing stood between "memory is tight" and "processes die at random": the kernel OOM killer only fires after an allocation has already failed, so a machine under pressure thrashes in reclaim until something breaks somewhere fatal — seen in practice as Hyprland taking SIGBUS mid-memcpy from a client shm pool that memory pressure was tearing down. Enable systemd-oomd, which keys on PSI stall time rather than free pages, and mark app.slice — and only app.slice — as its kill candidate. Hyprland runs in session.slice, so the compositor is structurally ineligible as a victim: oomd takes the app that caused the pressure and the session survives. Thresholds (50% stall over 20s, Fedora's desktop defaults) live in an oomd.conf.d drop-in; candidacy ships as a vendor drop-in under /usr/lib/systemd/user so existing users get it on package upgrade with no per-user seeding. New installs enable the daemon from enable-services.sh; a migration covers existing ones, restarting an already-running oomd so it doesn't keep stale thresholds until reboot. Deliberately not earlyoom: it triggers only when free RAM and free swap are both under threshold, so Omarchy's large, mostly-idle swapfiles keep it asleep through exactly this failure. Co-Authored-By: Claude Fable 5 --- default/systemd/user/app.slice.d/10-oomd.conf | 16 ++++++++++ docs/file-layout.md | 1 + etc/systemd/oomd.conf.d/10-omarchy.conf | 21 +++++++++++++ install/config/enable-services.sh | 4 +++ migrations/1785424256.sh | 31 +++++++++++++++++++ test/shell.d/systemd-test.sh | 31 +++++++++++++++++++ 6 files changed, 104 insertions(+) create mode 100644 default/systemd/user/app.slice.d/10-oomd.conf create mode 100644 etc/systemd/oomd.conf.d/10-omarchy.conf create mode 100644 migrations/1785424256.sh diff --git a/default/systemd/user/app.slice.d/10-oomd.conf b/default/systemd/user/app.slice.d/10-oomd.conf new file mode 100644 index 0000000000..f6b34e523a --- /dev/null +++ b/default/systemd/user/app.slice.d/10-oomd.conf @@ -0,0 +1,16 @@ +# Make user apps the only thing systemd-oomd is allowed to kill. +# +# Hyprland runs in session.slice, as wayland-wm@hyprland.desktop.service, while +# everything launched through uwsm-app lands in app.slice/app-*.scope. Marking +# only app.slice as a kill candidate means the compositor is structurally +# ineligible: oomd takes the browser or terminal that caused the pressure, and +# the session survives to show the notification about it. Setting this on +# user@.service instead would put the compositor back in the candidate pool. +# +# Swap kill is a backstop for the slower shape of the same problem, where swap +# fills before pressure spikes. It uses the global SwapUsedLimit (90%) from +# /etc/systemd/oomd.conf.d/10-omarchy.conf, which also carries the pressure +# thresholds. +[Slice] +ManagedOOMMemoryPressure=kill +ManagedOOMSwap=kill diff --git a/docs/file-layout.md b/docs/file-layout.md index f481d22979..29e8196aff 100644 --- a/docs/file-layout.md +++ b/docs/file-layout.md @@ -105,6 +105,7 @@ default/** ──► omarchy-settings /usr/share/omarchy ├─ xdg-terminal-exec/*.list /usr/share/xdg-terminal-exec/ ├─ applications/mimeapps.list /usr/share/applications/mimeapps.list ├─ systemd/user/*.{service,path} /usr/lib/systemd/user/ + ├─ systemd/user/app.slice.d/10-oomd.conf /usr/lib/systemd/user/app.slice.d/ ├─ systemd/system-sleep/unmount-fuse /usr/lib/systemd/system-sleep/ ├─ systemd/zram-generator.conf.d/90-omarchy.conf /usr/lib/systemd/zram-generator.conf.d/ ├─ fonts/omarchy/omarchy.ttf /usr/share/fonts/omarchy/ diff --git a/etc/systemd/oomd.conf.d/10-omarchy.conf b/etc/systemd/oomd.conf.d/10-omarchy.conf new file mode 100644 index 0000000000..7a45df1545 --- /dev/null +++ b/etc/systemd/oomd.conf.d/10-omarchy.conf @@ -0,0 +1,21 @@ +# Without an OOM daemon there is nothing between "memory is tight" and +# "processes start dying at random". The kernel OOM killer only fires once an +# allocation has already failed outright, which is far too late for the +# desktop: the machine thrashes in reclaim until something fails somewhere +# fatal. Seen in practice as Hyprland taking SIGBUS mid-memcpy from a Wayland +# client's shm pool, because the client was being torn down by the same memory +# pressure. One app misbehaving should not cost the whole session. +# +# systemd-oomd keys on PSI stall time rather than on free pages, so it acts +# while the machine is still thrashing instead of after an allocation failed. +# Deliberately not earlyoom: that fires only when free RAM *and* free swap are +# both under threshold, so a large, mostly-unused swapfile keeps it asleep +# through exactly this failure. +# +# 50% means half of a ten-second window was spent stalled on memory reclaim. +# Held for 20s straight, the desktop is already unusable and losing one app is +# the cheaper outcome. Which cgroups are eligible is set separately, in +# /usr/lib/systemd/user/app.slice.d/10-oomd.conf. +[OOM] +DefaultMemoryPressureDurationSec=20s +DefaultMemoryPressureLimit=50% diff --git a/install/config/enable-services.sh b/install/config/enable-services.sh index 2a1698e581..b7e2ba8b93 100644 --- a/install/config/enable-services.sh +++ b/install/config/enable-services.sh @@ -14,3 +14,7 @@ systemctl enable NetworkManager.service systemctl mask NetworkManager-wait-online.service systemctl enable power-profiles-daemon.service systemctl enable sddm.service +# Kill one runaway app scope instead of letting reclaim thrashing take the +# whole session down. [Install] pulls in systemd-oomd.socket via Also=, which +# is what the user manager reports app.slice candidacy over. +systemctl enable systemd-oomd.service diff --git a/migrations/1785424256.sh b/migrations/1785424256.sh new file mode 100644 index 0000000000..d1d2f4cea5 --- /dev/null +++ b/migrations/1785424256.sh @@ -0,0 +1,31 @@ +echo "Let systemd-oomd kill a runaway app instead of the whole session" + +# New installs get this from install/config/enable-services.sh. Existing ones +# have never had an OOM daemon, so nothing stands between memory pressure and +# the session falling over. + +as_root() { + if (( EUID == 0 )); then + "$@" + else + sudo "$@" + fi +} + +# Machine-wide, so a second user on the same box finds it already done. +if systemctl is-enabled --quiet systemd-oomd.service 2>/dev/null; then + # Already enabled means it started before the package delivered our + # thresholds in /etc/systemd/oomd.conf.d/, and oomd only reads that at + # startup. Restart so it doesn't run with stale limits until reboot. + as_root systemctl try-restart systemd-oomd.service >/dev/null 2>&1 || true +else + as_root systemctl enable --now systemd-oomd.service >/dev/null 2>&1 || + echo "Could not enable systemd-oomd.service; memory pressure will still take the session down." +fi + +# Pick up /usr/lib/systemd/user/app.slice.d/10-oomd.conf without waiting for +# the next login. That drop-in is what marks app.slice as a kill candidate; +# until the user manager reloads and reports it, oomd is running with nothing +# to act on. An `omarchy update` over SSH or from a TTY has no user manager to +# reload, and there the next graphical login picks it up on its own. +systemctl --user daemon-reload >/dev/null 2>&1 || true diff --git a/test/shell.d/systemd-test.sh b/test/shell.d/systemd-test.sh index 7bce6240d7..a232fd54ce 100755 --- a/test/shell.d/systemd-test.sh +++ b/test/shell.d/systemd-test.sh @@ -76,3 +76,34 @@ grep -F 'omarchy-fcitx5.service' "$first_run_units" >/dev/null || grep -F 'fcitx5' "$ROOT/default/hypr/autostart.lua" >/dev/null && fail "fcitx5 is autostarted from Hyprland; an unsupervised launch dies silently and takes every compose sequence with it" pass "fcitx5 runs supervised, so a lost input method comes back instead of killing XCompose until logout" + +oomd_slice="$ROOT/default/systemd/user/app.slice.d/10-oomd.conf" +grep -Fx 'ManagedOOMMemoryPressure=kill' "$oomd_slice" >/dev/null || + fail "nothing is a kill candidate, so systemd-oomd watches the machine thrash and never acts" +grep -Fx 'ManagedOOMSwap=kill' "$oomd_slice" >/dev/null || + fail "no swap backstop for the slower shape of the same failure" + +# Hyprland lives in session.slice/wayland-wm@hyprland.desktop.service. Marking +# any ancestor of that as a kill candidate puts the compositor back in the +# victim pool, which is the crash this whole thing exists to prevent. +candidates=$(grep -rlE '^ManagedOOM(MemoryPressure|Swap)=kill' "$ROOT/default/systemd" "$ROOT/etc/systemd" 2>/dev/null || true) +[[ $candidates == "$oomd_slice" ]] || + fail "systemd-oomd kill candidacy is set outside app.slice, which can select the compositor: $candidates" +pass "only user app scopes are systemd-oomd kill candidates" + +oomd_conf="$ROOT/etc/systemd/oomd.conf.d/10-omarchy.conf" +grep -Fx 'DefaultMemoryPressureLimit=50%' "$oomd_conf" >/dev/null || + fail "no pressure limit; the 60% default rides thrashing longer than a desktop stays usable" +grep -Fx 'DefaultMemoryPressureDurationSec=20s' "$oomd_conf" >/dev/null || + fail "no pressure duration set for the tightened limit" +pass "systemd-oomd acts on sustained memory stall" + +grep -Fx 'systemctl enable systemd-oomd.service' "$ROOT/install/config/enable-services.sh" >/dev/null || + fail "new installs ship the oomd drop-ins with the daemon that reads them disabled" + +oomd_migration=$(grep -rl 'systemd-oomd.service' "$ROOT/migrations" | head -n 1 || true) +[[ -n $oomd_migration ]] || + fail "existing installs never enable systemd-oomd; enable-services.sh only runs at install time" +grep -F 'systemctl --user daemon-reload' "$oomd_migration" >/dev/null || + fail "migration leaves the user manager unaware of app.slice candidacy until the next login" +pass "existing installs enable systemd-oomd and report app.slice without a relogin"