feat: tell the user when the Podman Machine runs out of memory and offer more

docs/getting-started/requirements.md

@@ -81,10 +81,10 @@ On first start `lerd` sizes the Podman Machine VM based on your host RAM so 8 GB
 | 9-31 GB  | 4 GB                  |
 | ≥ 32 GB  | 6 GB                  |
 
-The memory value is a ceiling, not a reservation: the VM only uses what your containers actually request. If sites slow down under load on an 8 GB host, bump the VM manually:
+The memory value is a ceiling, not a reservation: the VM only uses what your containers actually request. If the VM runs out of memory the dashboard says so and offers a restart with more, or you can do it from the terminal:
 
 ```bash
-podman machine stop
-podman machine set --memory 4096
-podman machine start
+lerd machine memory 6
 ```
+
+See [`lerd machine memory`](../usage/lifecycle.md#lerd-machine-memory-macos).

docs/usage/lifecycle.md

@@ -261,6 +261,22 @@ This command is macOS-only. On Linux podman writes to the host filesystem direct
 
 ---
 
+## `lerd machine memory` (macOS)
+
+```bash
+lerd machine memory          # has the VM run out of memory since it booted?
+lerd machine memory 6        # restart the VM with 6 GB, asks for confirmation first
+lerd machine memory 6 --yes  # skip the prompt
+```
+
+When the Podman Machine VM runs out of memory, its kernel kills whatever is using the most, usually MySQL or a PHP worker, and commands that need them hang or fail. lerd reads the VM's console log on the host and counts those kills since the VM last booted, so checking costs nothing inside the VM.
+
+With no argument the command reports the kills and suggests a size: 2 GB more than the VM has, but never past half of the Mac's memory. With a size in GB it stops the VM, sets its memory and runs `lerd start`, so every container is stopped for the minute or so that takes. Site data and databases live on the host and are untouched.
+
+The dashboard shows the same finding as a banner while the VM has had kills, with a button that does the restart, and `lerd doctor` reports it under Containers & Images. A size you set this way is kept: `lerd start` only ever raises the VM to its default, never lowers it.
+
+---
+
 ## Autostart on login
 
 Lerd can boot itself every time you log in. Autostart is a single switch over every lerd-owned systemd user unit on the machine:
@@ -352,6 +368,7 @@ Where a digest is given the download is checked against it and rejected on a mis
 | Full shutdown before a reinstall | `lerd quit` |
 | `lerd start` fails with an overlay / graph-driver storage error (macOS) | `lerd machine reset` |
 | lerd's VM is holding more host disk than it uses (macOS) | `lerd machine reclaim` |
+| lerd's VM keeps running out of memory (macOS) | `lerd machine memory` |
 | Verify everything's healthy | `lerd status` |
 | Update Composer / mise / fnm / mkcert to their pinned versions | `lerd tools:update` |
 | Uninstall a service entirely (data preserved) | `lerd service remove <name>` |

internal/cli/doctor.go

@@ -572,6 +572,17 @@ func runDoctorInto(w io.Writer, useColor bool) (DoctorReport, error) {
 	section = "Containers & Images"
 	fmt.Fprintln(w, "\n[Containers & Images]")
 
+	if platform.Current.UsesMachineVM {
+		if st := MachineMemoryStatus(); len(st.Kills) > 0 {
+			fail("Podman Machine memory", st.Summary(), st.Hint())
+			if st.SuggestedMiB > 0 {
+				rep.fixLast(manualFixWith("run `" + st.Hint() + "` (it restarts every container)"))
+			}
+		} else {
+			ok("Podman Machine has not run out of memory")
+		}
+	}
+
 	if detail, hint, healthy := nginxServiceFinding(nginxInstalled, nginxRunning); healthy {
 		ok("lerd-nginx service running")
 	} else {

internal/cli/machine.go

@@ -1,6 +1,12 @@
 package cli
 
-import "github.com/spf13/cobra"
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/geodro/lerd/internal/feedback"
+	"github.com/spf13/cobra"
+)
 
 // NewMachineCmd returns the machine parent command for Podman Machine
 // maintenance operations.
@@ -11,6 +17,7 @@ func NewMachineCmd() *cobra.Command {
 	}
 	cmd.AddCommand(newMachineResetCmd())
 	cmd.AddCommand(newMachineReclaimCmd())
+	cmd.AddCommand(newMachineMemoryCmd())
 	return cmd
 }
 
@@ -49,3 +56,45 @@ preserved; container images are rebuilt automatically on the next start.`,
 	cmd.Flags().BoolVarP(&assumeYes, "yes", "y", false, "Skip the confirmation prompt")
 	return cmd
 }
+
+func newMachineMemoryCmd() *cobra.Command {
+	var assumeYes bool
+	cmd := &cobra.Command{
+		Use:   "memory [GB]",
+		Short: "Show or set the Podman Machine VM's memory (macOS)",
+		Long: `Show whether the Podman Machine VM has run out of memory since it booted,
+or restart it with GB of memory. The restart stops every container and brings
+them back with lerd start; site data is untouched.`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(_ *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				printMachineMemory(MachineMemoryStatus())
+				return nil
+			}
+			gb, err := strconv.Atoi(args[0])
+			if err != nil {
+				return fmt.Errorf("%q is not a whole number of GB", args[0])
+			}
+			if !assumeYes && !confirmMachineRestart(gb) {
+				fmt.Println("Cancelled.")
+				return nil
+			}
+			return ResizeMachineMemory(int64(gb)*1024, nil)
+		},
+	}
+	cmd.Flags().BoolVarP(&assumeYes, "yes", "y", false, "Skip the confirmation prompt")
+	return cmd
+}
+
+func printMachineMemory(st MachineMemory) {
+	if len(st.Kills) == 0 {
+		fmt.Println("The Podman Machine has not run out of memory since it booted.")
+		return
+	}
+	fmt.Println("The Podman Machine " + st.Summary() + ".")
+	fmt.Println("Fix: " + st.Hint())
+}
+
+func confirmMachineRestart(gb int) bool {
+	return feedback.Confirm(fmt.Sprintf("Restart the Podman Machine with %d GB? Every container stops until lerd start brings them back.", gb), false)
+}

internal/cli/machine_memory_darwin.go

@@ -0,0 +1,76 @@
+//go:build darwin
+
+package cli
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/geodro/lerd/internal/podman"
+)
+
+// MachineMemoryStatus reports the processes the VM's kernel killed for lack of
+// memory since it booted, read from its serial console log on the host, so no
+// call reaches the VM. The memory sizes are only looked up when there were kills.
+func MachineMemoryStatus() MachineMemory {
+	name := selectedMachineName()
+	if name == "" {
+		return MachineMemory{}
+	}
+	b, err := os.ReadFile(machineConsoleLog(name))
+	if err != nil {
+		return MachineMemory{}
+	}
+	st := MachineMemory{Kills: podman.ParseOOMKills(string(b))}
+	if len(st.Kills) == 0 {
+		return st
+	}
+	out, err := machineQuery("machine", "inspect", "--format", "{{.Resources.Memory}}", name)
+	if err != nil {
+		return st
+	}
+	st.CurrentMiB, _ = strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64)
+	st.SuggestedMiB = raisedVMMemoryMiB(st.CurrentMiB, hostMemoryGiB())
+	return st
+}
+
+// machineConsoleLog is where podman keeps the VM's serial console, in the
+// per-user temp dir. launchd jobs do not always carry TMPDIR, hence getconf.
+func machineConsoleLog(name string) string {
+	dir := os.Getenv("TMPDIR")
+	if dir == "" {
+		out, _ := exec.Command("getconf", "DARWIN_USER_TEMP_DIR").Output()
+		dir = strings.TrimSpace(string(out))
+	}
+	return filepath.Join(dir, "podman", name+".log")
+}
+
+// ResizeMachineMemory restarts the VM with mib of memory, then runs the normal
+// start so every container comes back with its port forwards.
+func ResizeMachineMemory(mib int64, emit func(StartEvent), skip ...string) error {
+	name := selectedMachineName()
+	if name == "" {
+		return fmt.Errorf("no Podman Machine found; run `lerd start` to create one")
+	}
+	if ceiling := int64(hostMemoryGiB()) * 1024 / 2; mib < 2048 || mib > ceiling {
+		return fmt.Errorf("%d MB is outside 2048 to %d MB, half of this Mac's memory", mib, ceiling)
+	}
+	release, err := podman.LockLifecycle()
+	if err != nil {
+		return fmt.Errorf("lifecycle lock: %w", err)
+	}
+	if emit != nil {
+		emit(StartEvent{Phase: "step", Step: "machine"})
+	}
+	_ = runMachineStreaming(machineStopTimeout, "machine", "stop", name)
+	err = runMachineStreaming(machineQueryTimeout, "machine", "set", "--memory", strconv.FormatInt(mib, 10), name)
+	release()
+	if err != nil {
+		return fmt.Errorf("podman machine set --memory: %w", err)
+	}
+	return RunStart(emit, skip...)
+}

internal/cli/machine_memory_other.go

@@ -0,0 +1,13 @@
+//go:build !darwin
+
+package cli
+
+import "fmt"
+
+// MachineMemoryStatus is empty off macOS: there is no Podman Machine VM whose
+// memory lerd sizes.
+func MachineMemoryStatus() MachineMemory { return MachineMemory{} }
+
+func ResizeMachineMemory(int64, func(StartEvent), ...string) error {
+	return fmt.Errorf("resizing the Podman Machine is only supported on macOS")
+}

internal/cli/podman_memory.go

@@ -1,5 +1,10 @@
 package cli
 
+import (
+	"fmt"
+	"strings"
+)
+
 // recommendedVMMemoryMiB picks Podman Machine memory based on host RAM and
 // the active worker runtime mode. 8 GB is the minimum host RAM tier; anything
 // below (including detection failures) uses the 8 GB values.
@@ -33,3 +38,36 @@ func recommendedVMMemoryMiB(hostMemoryGiB int, execMode bool) int64 {
 		return 3072
 	}
 }
+
+// raisedVMMemoryMiB is the size to offer a VM that ran out of memory: 2 GB
+// more, but never past half the host's RAM so macOS keeps room of its own.
+// 0 when there is nothing more to give or the host size is unknown.
+func raisedVMMemoryMiB(currentMiB int64, hostMemoryGiB int) int64 {
+	ceiling := int64(hostMemoryGiB) * 1024 / 2
+	next := min(currentMiB+2048, ceiling)
+	if next <= currentMiB {
+		return 0
+	}
+	return next
+}
+
+// MachineMemory is the Podman Machine's out-of-memory record since it booted.
+type MachineMemory struct {
+	Kills        []string `json:"kills"`
+	CurrentMiB   int64    `json:"current_mib,omitempty"`
+	SuggestedMiB int64    `json:"suggested_mib,omitempty"` // 0: nothing more to give
+}
+
+// Summary says how often the VM ran out of memory and what the kernel killed.
+func (st MachineMemory) Summary() string {
+	return fmt.Sprintf("ran out of memory %d times since it booted (%s)", len(st.Kills), strings.Join(dedupeStrings(st.Kills), ", "))
+}
+
+// Hint is the command that gives the VM more memory, or what to do instead
+// when the host has none left to give.
+func (st MachineMemory) Hint() string {
+	if st.SuggestedMiB == 0 {
+		return "this Mac has no more memory to give it; stop sites or workers you are not using"
+	}
+	return fmt.Sprintf("lerd machine memory %d", st.SuggestedMiB/1024)
+}

internal/cli/podman_memory_test.go

@@ -39,3 +39,21 @@ func TestRecommendedVMMemoryMiB(t *testing.T) {
 		})
 	}
 }
+
+func TestRaisedVMMemoryMiB(t *testing.T) {
+	cases := []struct {
+		current int64
+		hostGiB int
+		want    int64
+	}{
+		{4096, 48, 6144}, // room to spare: 2 GB more
+		{4096, 10, 5120}, // capped at half the host
+		{8192, 16, 0},    // already at half the host
+		{3072, 0, 0},     // host size unknown: offer nothing
+	}
+	for _, c := range cases {
+		if got := raisedVMMemoryMiB(c.current, c.hostGiB); got != c.want {
+			t.Errorf("raisedVMMemoryMiB(%d, %d) = %d, want %d", c.current, c.hostGiB, got, c.want)
+		}
+	}
+}

internal/cli/startstop_darwin.go

@@ -408,7 +408,7 @@ func startPodmanMachineIfNeeded() error {
 				if needsMemory {
 					if hostGiB > 0 && hostGiB <= 8 {
 						feedback.Line(fmt.Sprintf("Host has %d GB RAM; setting Podman Machine to %d MB (tight but workable)…", hostGiB, targetMemoryMiB))
-						feedback.Note("If sites slow down under load, run: podman machine set --memory 4096")
+						feedback.Note("If the VM runs out of memory, run: lerd machine memory 4")
 					} else {
 						feedback.Line(fmt.Sprintf("Setting Podman Machine memory to %d MB…", targetMemoryMiB))
 					}

internal/podman/machine_oom.go

@@ -0,0 +1,35 @@
+package podman
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+var (
+	kernelStamp = regexp.MustCompile(`\[\s*(\d+)\.\d+\]`)
+	oomKill     = regexp.MustCompile(`Out of memory: Killed process \d+ \(([^)]+)\)`)
+)
+
+// ParseOOMKills returns the processes the VM's kernel killed for lack of memory
+// since its last boot, read from the machine's serial console log. The log can
+// span boots, so a kernel uptime stamp going backwards starts the count over.
+func ParseOOMKills(log string) []string {
+	var kills []string
+	last := int64(-1)
+	for _, line := range strings.Split(log, "\n") {
+		m := kernelStamp.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		stamp, _ := strconv.ParseInt(m[1], 10, 64)
+		if stamp < last {
+			kills = nil
+		}
+		last = stamp
+		if k := oomKill.FindStringSubmatch(line); k != nil {
+			kills = append(kills, k[1])
+		}
+	}
+	return kills
+}

internal/podman/machine_oom_test.go

@@ -0,0 +1,32 @@
+package podman
+
+import (
+	"reflect"
+	"testing"
+)
+
+// Trimmed from a real applehv console log: the kernel's uptime stamp starting
+// over marks a reboot, and only the kills after it are this boot's.
+const oomConsoleLog = `[    0.000000] Booting Linux on physical CPU 0x0000000000 [0x610f0000]
+[708961.947962] Out of memory: Killed process 1318 (dbus-broker-lau) total-vm:9292kB, anon-rss:276kB
+[709000.465360] Out of memory: Killed process 2357719 (mysqld) total-vm:2178448kB, anon-rss:435684kB
+[    0.000000] Booting Linux on physical CPU 0x0000000000 [0x610f0000]
+[    2.114023] systemd[1]: Detected virtualization apple.
+[root@localhost ~]# [  239.106785] Out of memory: Killed process 4157 (mysqld) total-vm:2112916kB, anon-rss:456404kB
+[  919.398937] Out of memory: Killed process 1144 (systemd) total-vm:24620kB, anon-rss:6588kB
+[  948.967036] Out of memory: Killed process 3952 (php-fpm) total-vm:268840kB, anon-rss:64160kB
+`
+
+func TestParseOOMKills_countsOnlyTheCurrentBoot(t *testing.T) {
+	got := ParseOOMKills(oomConsoleLog)
+	want := []string{"mysqld", "systemd", "php-fpm"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("ParseOOMKills = %v, want %v", got, want)
+	}
+}
+
+func TestParseOOMKills_noneOnAHealthyBoot(t *testing.T) {
+	if got := ParseOOMKills("[    0.000000] Booting Linux\n[    2.1] systemd[1]: ok\n"); len(got) != 0 {
+		t.Errorf("ParseOOMKills = %v, want none", got)
+	}
+}