feat: tell the user when the Podman Machine runs out of memory and offer more

docs/getting-started/requirements.md

@@ -81,10 +81,10 @@ On first start `lerd` sizes the Podman Machine VM based on your host RAM so 8 GB
 | 9-31 GB  | 4 GB                  |
 | ≥ 32 GB  | 6 GB                  |
 
-The memory value is a ceiling, not a reservation: the VM only uses what your containers actually request. If sites slow down under load on an 8 GB host, bump the VM manually:
+The memory value is a ceiling, not a reservation: the VM only uses what your containers actually request. If the VM runs out of memory the dashboard says so and offers a restart with more, or you can do it from the terminal:
 
 ```bash
-podman machine stop
-podman machine set --memory 4096
-podman machine start
+lerd machine memory 6
 ```
+
+See [`lerd machine memory`](../usage/lifecycle.md#lerd-machine-memory-macos).

docs/usage/lifecycle.md

@@ -261,6 +261,22 @@ This command is macOS-only. On Linux podman writes to the host filesystem direct
 
 ---
 
+## `lerd machine memory` (macOS)
+
+```bash
+lerd machine memory          # has the VM run out of memory since it booted?
+lerd machine memory 6        # restart the VM with 6 GB, asks for confirmation first
+lerd machine memory 6 --yes  # skip the prompt
+```
+
+When the Podman Machine VM runs out of memory, its kernel kills whatever is using the most, usually MySQL or a PHP worker, and commands that need them hang or fail. lerd reads the VM's console log on the host and counts those kills since the VM last booted, so checking costs nothing inside the VM.
+
+With no argument the command reports the kills and suggests a size: 2 GB more than the VM has, but never past half of the Mac's memory. With a size in GB it stops the VM, sets its memory and runs `lerd start`, so every container is stopped for the minute or so that takes. Site data and databases live on the host and are untouched.
+
+The dashboard shows the same finding as a banner while the VM has had kills, with a button that does the restart, and `lerd doctor` reports it under Containers & Images. A size you set this way is kept: `lerd start` only ever raises the VM to its default, never lowers it.
+
+---
+
 ## Autostart on login
 
 Lerd can boot itself every time you log in. Autostart is a single switch over every lerd-owned systemd user unit on the machine:
@@ -352,6 +368,7 @@ Where a digest is given the download is checked against it and rejected on a mis
 | Full shutdown before a reinstall | `lerd quit` |
 | `lerd start` fails with an overlay / graph-driver storage error (macOS) | `lerd machine reset` |
 | lerd's VM is holding more host disk than it uses (macOS) | `lerd machine reclaim` |
+| lerd's VM keeps running out of memory (macOS) | `lerd machine memory` |
 | Verify everything's healthy | `lerd status` |
 | Update Composer / mise / fnm / mkcert to their pinned versions | `lerd tools:update` |
 | Uninstall a service entirely (data preserved) | `lerd service remove <name>` |

internal/cli/doctor.go

@@ -595,6 +595,19 @@ func runDoctorInto(w io.Writer, useColor bool) (DoctorReport, error) {
 	section = "Containers & Images"
 	fmt.Fprintln(w, "\n[Containers & Images]")
 
+	if platform.Current.UsesMachineVM {
+		if st, err := MachineMemoryStatus(); err != nil {
+			warn("Podman Machine memory", "could not check: "+err.Error())
+		} else if len(st.Kills) > 0 {
+			fail("Podman Machine memory", st.Summary(), st.Hint())
+			if st.SuggestedMiB > 0 {
+				rep.fixLast(manualFixWith("run `" + st.Hint() + "` (it restarts every container)"))
+			}
+		} else {
+			ok("Podman Machine has not run out of memory")
+		}
+	}
+
 	if detail, hint, healthy := nginxServiceFinding(nginxInstalled, nginxRunning); healthy {
 		ok("lerd-nginx service running")
 	} else {

internal/cli/machine.go

@@ -1,6 +1,12 @@
 package cli
 
-import "github.com/spf13/cobra"
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/geodro/lerd/internal/feedback"
+	"github.com/spf13/cobra"
+)
 
 // NewMachineCmd returns the machine parent command for Podman Machine
 // maintenance operations.
@@ -11,6 +17,7 @@ func NewMachineCmd() *cobra.Command {
 	}
 	cmd.AddCommand(newMachineResetCmd())
 	cmd.AddCommand(newMachineReclaimCmd())
+	cmd.AddCommand(newMachineMemoryCmd())
 	return cmd
 }
 
@@ -49,3 +56,49 @@ preserved; container images are rebuilt automatically on the next start.`,
 	cmd.Flags().BoolVarP(&assumeYes, "yes", "y", false, "Skip the confirmation prompt")
 	return cmd
 }
+
+func newMachineMemoryCmd() *cobra.Command {
+	var assumeYes bool
+	cmd := &cobra.Command{
+		Use:   "memory [GB]",
+		Short: "Show or set the Podman Machine VM's memory (macOS)",
+		Long: `Show whether the Podman Machine VM has run out of memory since it booted,
+or restart it with GB of memory. The restart stops every container and brings
+them back with lerd start; site data is untouched.`,
+		Args: cobra.MaximumNArgs(1),
+		RunE: func(_ *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				st, err := MachineMemoryStatus()
+				if err != nil {
+					return fmt.Errorf("could not check the Podman Machine's memory: %w", err)
+				}
+				printMachineMemory(st)
+				return nil
+			}
+			gb, err := strconv.Atoi(args[0])
+			if err != nil {
+				return fmt.Errorf("%q is not a whole number of GB", args[0])
+			}
+			if !assumeYes && !confirmMachineRestart(gb) {
+				fmt.Println("Cancelled.")
+				return nil
+			}
+			return ResizeMachineMemory(int64(gb)*1024, nil)
+		},
+	}
+	cmd.Flags().BoolVarP(&assumeYes, "yes", "y", false, "Skip the confirmation prompt")
+	return cmd
+}
+
+func printMachineMemory(st MachineMemory) {
+	if len(st.Kills) == 0 {
+		fmt.Println("The Podman Machine has not run out of memory since it booted.")
+		return
+	}
+	fmt.Println("The Podman Machine " + st.Summary() + ".")
+	fmt.Println("Fix: " + st.Hint())
+}
+
+func confirmMachineRestart(gb int) bool {
+	return feedback.Confirm(fmt.Sprintf("Restart the Podman Machine with %d GB? Every container stops until lerd start brings them back.", gb), false)
+}

internal/cli/machine_memory_darwin.go

@@ -0,0 +1,84 @@
+//go:build darwin
+
+package cli
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/geodro/lerd/internal/podman"
+)
+
+// MachineMemoryStatus reports the processes the VM's kernel killed for lack of
+// memory since it booted, read from its serial console log on the host, so no
+// call reaches the VM. The memory sizes are only looked up when there were kills.
+// An error means the check could not run, which must not read as a healthy VM.
+func MachineMemoryStatus() (MachineMemory, error) {
+	name, err := selectedMachine()
+	if err != nil {
+		return MachineMemory{}, fmt.Errorf("listing Podman Machines: %w", err)
+	}
+	if name == "" {
+		return MachineMemory{}, nil
+	}
+	b, err := os.ReadFile(machineConsoleLog(name))
+	if err != nil {
+		return MachineMemory{}, fmt.Errorf("reading the VM's console log: %w", err)
+	}
+	st := MachineMemory{Kills: podman.ParseOOMKills(string(b))}
+	if len(st.Kills) == 0 {
+		return st, nil
+	}
+	out, err := machineQuery("machine", "inspect", "--format", "{{.Resources.Memory}}", name)
+	if err != nil {
+		return st, nil
+	}
+	st.CurrentMiB, _ = strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64)
+	st.SuggestedMiB = raisedVMMemoryMiB(st.CurrentMiB, hostMemoryGiB())
+	return st, nil
+}
+
+// machineConsoleLog is where podman keeps the VM's serial console, in the
+// per-user temp dir. launchd jobs do not always carry TMPDIR, hence getconf.
+func machineConsoleLog(name string) string {
+	dir := os.Getenv("TMPDIR")
+	if dir == "" {
+		out, _ := exec.Command("getconf", "DARWIN_USER_TEMP_DIR").Output()
+		dir = strings.TrimSpace(string(out))
+	}
+	return filepath.Join(dir, "podman", name+".log")
+}
+
+// ResizeMachineMemory restarts the VM with mib of memory, then runs the normal
+// start so every container comes back with its port forwards.
+func ResizeMachineMemory(mib int64, emit func(StartEvent), skip ...string) error {
+	name := selectedMachineName()
+	if name == "" {
+		return fmt.Errorf("no Podman Machine found; run `lerd start` to create one")
+	}
+	if ceiling := int64(hostMemoryGiB()) * 1024 / 2; mib < 2048 || mib > ceiling {
+		return fmt.Errorf("%d MB is outside 2048 to %d MB, half of this Mac's memory", mib, ceiling)
+	}
+	release, err := podman.LockLifecycle()
+	if err != nil {
+		return fmt.Errorf("lifecycle lock: %w", err)
+	}
+	if emit != nil {
+		emit(StartEvent{Phase: "step", Step: "machine"})
+	}
+	_ = runMachineStreaming(machineStopTimeout, "machine", "stop", name)
+	setErr := runMachineStreaming(machineQueryTimeout, "machine", "set", "--memory", strconv.FormatInt(mib, 10), name)
+	release()
+	// Start even when the set failed: the VM is already down, and leaving it
+	// there takes every site with it.
+	startErr := RunStart(emit, skip...)
+	if setErr != nil {
+		return errors.Join(fmt.Errorf("podman machine set --memory: %w", setErr), startErr)
+	}
+	return startErr
+}

internal/cli/machine_memory_darwin_test.go

@@ -0,0 +1,78 @@
+//go:build darwin
+
+package cli
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+// TestMachineListHelperProcess is the child the fake machineCmdContext spawns:
+// it prints GO_MACHINE_LIST_OUT, or fails when that is unset.
+func TestMachineListHelperProcess(t *testing.T) {
+	if os.Getenv("GO_WANT_MACHINE_LIST_HELPER") != "1" {
+		return
+	}
+	out, ok := os.LookupEnv("GO_MACHINE_LIST_OUT")
+	if !ok {
+		os.Exit(1)
+	}
+	fmt.Print(out)
+	os.Exit(0)
+}
+
+func fakeMachineList(t *testing.T, out *string) {
+	t.Helper()
+	prev := machineCmdContext
+	t.Cleanup(func() { machineCmdContext = prev })
+	machineCmdContext = func(ctx context.Context, args ...string) *exec.Cmd {
+		cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=TestMachineListHelperProcess", "--")
+		cmd.Env = append(os.Environ(), "GO_WANT_MACHINE_LIST_HELPER=1")
+		if out != nil {
+			cmd.Env = append(cmd.Env, "GO_MACHINE_LIST_OUT="+*out)
+		}
+		return cmd
+	}
+}
+
+// A machine list that fails must not read as a VM with no kills.
+func TestMachineMemoryStatusReportsAFailedMachineList(t *testing.T) {
+	fakeMachineList(t, nil)
+
+	if _, err := MachineMemoryStatus(); err == nil {
+		t.Fatal("expected an error when the machine list fails")
+	}
+}
+
+// An unreadable console log means the check could not run, not that it passed.
+func TestMachineMemoryStatusReportsAMissingConsoleLog(t *testing.T) {
+	out := "lerd*\n"
+	fakeMachineList(t, &out)
+	t.Setenv("TMPDIR", t.TempDir())
+
+	if _, err := MachineMemoryStatus(); err == nil {
+		t.Fatal("expected an error when the console log is missing")
+	}
+}
+
+func TestMachineMemoryStatusReadsTheConsoleLog(t *testing.T) {
+	out := "lerd*\n"
+	fakeMachineList(t, &out)
+	dir := t.TempDir()
+	t.Setenv("TMPDIR", dir)
+	if err := os.MkdirAll(filepath.Join(dir, "podman"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "podman", "lerd.log"), []byte("booted\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	st, err := MachineMemoryStatus()
+	if err != nil || len(st.Kills) != 0 {
+		t.Fatalf("got %+v, %v; want no kills and no error", st, err)
+	}
+}

internal/cli/machine_memory_other.go

@@ -0,0 +1,13 @@
+//go:build !darwin
+
+package cli
+
+import "fmt"
+
+// MachineMemoryStatus is empty off macOS: there is no Podman Machine VM whose
+// memory lerd sizes.
+func MachineMemoryStatus() (MachineMemory, error) { return MachineMemory{}, nil }
+
+func ResizeMachineMemory(int64, func(StartEvent), ...string) error {
+	return fmt.Errorf("resizing the Podman Machine is only supported on macOS")
+}

internal/cli/machine_restart_heal_darwin.go

@@ -21,12 +21,18 @@ func machineLastUpFile() string {
 
 // selectedMachineName mirrors ensurePodmanMachineRunning's selection order
 // (default-marked, else first) so inspect targets the VM lerd actually
-// uses. Returns "" when no machine exists yet.
+// uses. Returns "" when no machine exists yet or the list fails.
 func selectedMachineName() string {
-	out, err := podman.Cmd("machine", "list",
-		"--format", "{{.Name}}").Output()
+	name, _ := selectedMachine()
+	return name
+}
+
+// selectedMachine is selectedMachineName keeping the list's error, bounded so a
+// wedged VM cannot hang the caller.
+func selectedMachine() (string, error) {
+	out, err := machineQuery("machine", "list", "--format", "{{.Name}}")
 	if err != nil {
-		return ""
+		return "", err
 	}
 	var first, def string
 	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
@@ -44,9 +50,9 @@ func selectedMachineName() string {
 		}
 	}
 	if def != "" {
-		return def
+		return def, nil
 	}
-	return first
+	return first, nil
 }
 
 // currentMachineLastUp returns the LastUp of lerd's Podman Machine, or ""