feat: PAUSED+PAGED app tier — freeze idle apps, page to swap, wake in ms (#81)

agents.md

@@ -24,8 +24,10 @@ shard_core/
     management/         Management API auth (hosted shards only)
   service/            → Business logic
     app_installation/   App install/uninstall/reinstall + background worker queue
-    app_lifecycle.py    Auto-start/stop containers based on idle time
-    app_tools.py        Docker CLI wrapper functions
+    app_lifecycle.py    Two-tier idle control: RUNNING -> PAUSED -> STOPPED, PSI-driven LRU demotion, wake-on-request
+    app_tools.py        Docker CLI wrapper functions (start/stop/pause/unpause/down)
+    memory_pressure.py  PSI parsing (/host/pressure/memory), cgroup v2 memory.reclaim page-out
+    pause_metrics.py    In-memory pause-tier telemetry accumulators (transitions, latencies, PSI snapshots)
     pairing.py          Terminal pairing (JWT creation, code generation)
     backup.py           Azure Blob Storage backup via rclone
     peer.py             Peer shard management
@@ -77,7 +79,7 @@ Tables: `identities`, `terminals`, `installed_apps`, `peers`, `backups`, `tours`
 ### Background Tasks
 Started at app lifespan startup, stopped at shutdown:
 - `InstallationWorker` — async task queue for app install/uninstall
-- `PeriodicTask(control_apps, 30s)` — auto-start/stop app containers
+- `PeriodicTask(control_apps, 30s)` — app idle lifecycle. With `apps.lifecycle.pause_enabled` (default off): RUNNING pauses after `idle_for_pause` (cgroup freeze + page-out to swap), PAUSED stops after `idle_for_stop`, and high memory PSI demotes the LRU app one tier per cycle. Flag off: legacy stop-only
 - `PeriodicTask(update_disk_space, 30s)` — disk monitoring
 - `CronTask(start_backup, "0 3 * * *")` — daily backup with random delay
 - `CronTask(docker_prune_images, daily)` — image cleanup

config.toml

@@ -34,6 +34,10 @@ password = "bk11Ws.8OGakl-~t.7bKh5l9EYXTbHw.0T"
 
 [apps.lifecycle]
 refresh_interval = 10
+default_idle_for_pause = 60
+default_idle_for_stop = 10800
+psi_threshold = 10.0
+pause_enabled = false
 
 [apps]
 initial_apps = ["filebrowser", "immich", "paperless-ngx"]

shard_core/data_model/app_meta.py

@@ -4,13 +4,13 @@
 from pathlib import Path as FilePath
 from typing import Optional, List, Dict, Union
 
-from pydantic import BaseModel, model_validator, field_validator
+from pydantic import BaseModel, model_validator
 
 from shard_core.data_model import app_meta_migration
 from shard_core.settings import settings
 from shard_core.util import signals
 
-CURRENT_VERSION = "1.2"
+CURRENT_VERSION = "1.3"
 
 
 class InstallationReason(str, Enum):
@@ -32,6 +32,7 @@ class Status(str, Enum):
     INSTALLING = "installing"
     STOPPED = "stopped"
     RUNNING = "running"
+    PAUSED = "paused"
     UNINSTALLATION_QUEUED = "uninstallation_queued"
     UNINSTALLING = "uninstalling"
     REINSTALLATION_QUEUED = "reinstallation_queued"
@@ -89,25 +90,36 @@ class Entrypoint(BaseModel):
 
 class Lifecycle(BaseModel):
     always_on: bool = False
-    idle_time_for_shutdown: Optional[int] = None
-
-    @field_validator("idle_time_for_shutdown")
-    @classmethod
-    def validate_idle_time_for_shutdown(cls, v):
-        if v and v < 5:
-            raise ValueError(f"idle_time_for_shutdown must be at least 5, was {v}")
-        return v
+    skip_pause: bool = False
+    idle_for_pause: int | None = None  # None = use global default
+    idle_for_stop: int | None = None  # None = use global default
 
     @model_validator(mode="after")
-    def validate_exclusivity(self):
-        if self.always_on and self.idle_time_for_shutdown is not None:
+    def validate_combinations(self):
+        if self.always_on and (
+            self.skip_pause
+            or self.idle_for_pause is not None
+            or self.idle_for_stop is not None
+        ):
+            raise ValueError(
+                "if always_on is true, no other lifecycle field may be set"
+            )
+        if self.skip_pause and self.idle_for_pause is not None:
+            raise ValueError("skip_pause and idle_for_pause are mutually exclusive")
+        if self.idle_for_pause is not None and self.idle_for_pause < 5:
             raise ValueError(
-                "if always_on is true, idle_time_for_shutdown must not be set"
+                f"idle_for_pause must be at least 5, was {self.idle_for_pause}"
             )
-        if not self.always_on and self.idle_time_for_shutdown is None:
+        if self.idle_for_stop is not None and self.idle_for_stop < 5:
             raise ValueError(
-                "if always_on is false or not set, idle_time_for_shutdown must be set"
+                f"idle_for_stop must be at least 5, was {self.idle_for_stop}"
             )
+        if (
+            self.idle_for_pause is not None
+            and self.idle_for_stop is not None
+            and self.idle_for_pause >= self.idle_for_stop
+        ):
+            raise ValueError("idle_for_pause must be less than idle_for_stop")
         return self
 
 
@@ -121,7 +133,7 @@ class AppMeta(BaseModel):
     icon: str
     entrypoints: List[Entrypoint]
     paths: Dict[str, Path]
-    lifecycle: Lifecycle = Lifecycle(idle_time_for_shutdown=60)
+    lifecycle: Lifecycle = Lifecycle()
     minimum_portal_size: VMSize = VMSize.XS
     store_info: Optional[StoreInfo] = None
 

shard_core/data_model/app_meta_migration.py

@@ -9,7 +9,23 @@ def migrate_1_1_to_1_2(values):
     return values
 
 
+def migrate_1_2_to_1_3(values):
+    # Lifecycle rework for the PAUSED+PAGED tier: idle_time_for_shutdown
+    # becomes idle_for_stop; idle_for_pause stays unset and falls back to the
+    # global default. App-repository files are not regenerated — this runs on
+    # every read of a v1.2 app_meta.json.
+    lifecycle = values.get("lifecycle") or {}
+    if not lifecycle.get("always_on"):
+        idle_time_for_shutdown = lifecycle.pop("idle_time_for_shutdown", None)
+        if idle_time_for_shutdown is not None:
+            lifecycle["idle_for_stop"] = idle_time_for_shutdown
+    values["lifecycle"] = lifecycle
+    values["v"] = "1.3"
+    return values
+
+
 migrations = {
     "1.0": migrate_1_0_to_1_1,
     "1.1": migrate_1_1_to_1_2,
+    "1.2": migrate_1_2_to_1_3,
 }

shard_core/data_model/backend/telemetry_model.py

@@ -1,11 +1,32 @@
 # DO NOT MODIFY - copied from freeshard-controller
 
 import datetime
+from typing import Dict, List, Optional
 
 from pydantic import BaseModel
 
 
+class PauseTierTelemetry(BaseModel):
+    """Metrics from the shard_core PAUSED+PAGED app tier (FreeshardBase/freeshard#81).
+
+    Sent by shards with the pause tier enabled; absent (None) from older shards
+    and from shards with the feature flag off.
+    """
+
+    # per-app transition counts, e.g. {"immich": {"running_to_paused": 3, "paused_to_running": 2}}
+    transitions: Dict[str, Dict[str, int]] = {}
+    pause_latency_ms_p50: Optional[float] = None
+    pause_latency_ms_p95: Optional[float] = None
+    unpause_latency_ms_p50: Optional[float] = None
+    unpause_latency_ms_p95: Optional[float] = None
+    # PSI `some avg10` sampled at each control cycle within the reporting interval
+    psi_some_avg10_snapshots: List[float] = []
+    swap_total_kib: Optional[int] = None
+    swap_free_kib: Optional[int] = None
+
+
 class Telemetry(BaseModel):
     start_time: datetime.datetime
     end_time: datetime.datetime
     no_of_requests: int
+    pause_tier: Optional[PauseTierTelemetry] = None

shard_core/service/app_installation/worker.py

@@ -17,6 +17,7 @@
     docker_create_app_containers,
     docker_stop_app,
     docker_shutdown_app,
+    docker_unpause_app,
 )
 from shard_core.settings import settings
 from shard_core.util import signals
@@ -123,6 +124,12 @@ async def _install_app_from_existing_zip(app_name: str):
 
 async def _uninstall_app(app_name: str):
     try:
+        installed_app = await get_app_from_db(app_name)
+        if installed_app.status == Status.PAUSED:
+            # unfreeze while the status still says PAUSED — once it flips to
+            # UNINSTALLING nothing knows the containers are frozen, and a
+            # frozen stack can be neither stopped nor removed
+            await docker_unpause_app(app_name)
         await update_app_status(app_name, Status.UNINSTALLING)
     except KeyError:
         log.warning(f"during uninstallation of {app_name}: app not found in database")

shard_core/service/app_lifecycle.py

@@ -1,25 +1,31 @@
 import asyncio
 import logging
 import time
-from typing import Dict
+from typing import Dict, List
 
 from shard_core.database.connection import db_conn
 from shard_core.database import installed_apps as db_installed_apps
 from shard_core.data_model.app_meta import InstalledApp, Status
-from shard_core.service import disk
+from shard_core.service import disk, memory_pressure, pause_metrics
 from shard_core.service.app_tools import (
     docker_start_app,
     docker_stop_app,
+    docker_pause_app,
+    docker_unpause_app,
     get_app_metadata,
     size_is_compatible,
 )
+from shard_core.settings import settings
 from shard_core.util import signals
 
 log = logging.getLogger(__name__)
 
 last_access_dict: Dict[str, float] = dict()
 background_tasks = set()
 
+# Pressure demotion never touches an app accessed within this window (seconds).
+RECENT_ACCESS_GRACE = 5
+
 
 @signals.on_request_to_app.connect
 async def ensure_app_is_running(app: InstalledApp):
@@ -29,38 +35,113 @@ async def ensure_app_is_running(app: InstalledApp):
     if await size_is_compatible(app_meta.minimum_portal_size):
         global last_access_dict
         last_access_dict[app.name] = time.time()
-        task = asyncio.create_task(
-            docker_start_app(app.name), name=f"ensure {app.name} is running"
-        )
+        # PAUSED wakes via unpause (ms to ~2s), everything else via the legacy
+        # start path. Calling docker_unpause_app directly also dodges the
+        # global 5s throttle on docker_start_app, which would silently drop
+        # the wake. This dispatch stays active even with pause_enabled=false,
+        # so already-paused apps still wake after a backout.
+        if app.status == Status.PAUSED:
+            coro = docker_unpause_app(app.name)
+        else:
+            coro = docker_start_app(app.name)
+        task = asyncio.create_task(coro, name=f"ensure {app.name} is running")
         background_tasks.add(task)
         task.add_done_callback(background_tasks.discard)
 
 
 async def control_apps():
+    lifecycle_settings = settings().apps.lifecycle
+    pause_enabled = lifecycle_settings.pause_enabled
+    psi = memory_pressure.read_memory_pressure() if pause_enabled else 0.0
+    pressure_high = pause_enabled and psi > lifecycle_settings.psi_threshold
+    if pause_enabled:
+        pause_metrics.record_psi_snapshot(psi)
+
     async with db_conn() as conn:
         all_apps = await db_installed_apps.get_all(conn)
     installed_apps = [
         InstalledApp.model_validate(a)
         for a in all_apps
         if a["status"] not in (Status.INSTALLATION_QUEUED, Status.INSTALLING)
     ]
-    tasks = [_control_app(app.name) for app in installed_apps]
+    tasks = [_control_app_time(app, pause_enabled) for app in installed_apps]
     await asyncio.gather(*tasks)
+    if pressure_high:
+        log.info(f"memory pressure high (PSI some avg10 = {psi}), demoting LRU app")
+        await _demote_lru(installed_apps)
 
 
-async def _control_app(name: str):
-    global last_access_dict
-    app_meta = get_app_metadata(name)
+async def _control_app_time(app: InstalledApp, pause_enabled: bool):
+    app_meta = get_app_metadata(app.name)
 
+    # Low disk stops every app, always_on included — a paused or running app
+    # can otherwise fill the disk completely.
     if disk.current_disk_usage.disk_space_low:
-        await docker_stop_app(name)
+        await docker_stop_app(app.name)
         return
 
     if app_meta.lifecycle.always_on:
-        if await size_is_compatible(app_meta.minimum_portal_size):
-            await docker_start_app(app_meta.name)
-    else:
-        last_access = last_access_dict.get(app_meta.name, 0.0)
-        idle_time_for_shutdown = app_meta.lifecycle.idle_time_for_shutdown
-        if last_access < time.time() - idle_time_for_shutdown:
-            await docker_stop_app(app_meta.name)
+        if app.status != Status.RUNNING and await size_is_compatible(
+            app_meta.minimum_portal_size
+        ):
+            await docker_start_app(app.name)
+        return
+
+    idle = time.time() - last_access_dict.get(app.name, 0.0)
+    t2 = (
+        app_meta.lifecycle.idle_for_stop
+        or settings().apps.lifecycle.default_idle_for_stop
+    )
+
+    # Feature flag off, or app opts out of the pause tier: legacy stop-only.
+    if not pause_enabled or app_meta.lifecycle.skip_pause:
+        if app.status == Status.RUNNING and idle >= t2:
+            await docker_stop_app(app.name)
+        return
+
+    t1 = (
+        app_meta.lifecycle.idle_for_pause
+        or settings().apps.lifecycle.default_idle_for_pause
+    )
+    if app.status == Status.RUNNING and idle >= t1:
+        await docker_pause_app(app.name)
+    elif app.status == Status.PAUSED and idle >= t2:
+        await docker_stop_app(app.name)
+
+
+async def _demote_lru(apps: List[InstalledApp]):
+    """Demote the least-recently-used demotable app one tier.
+
+    RUNNING apps are considered first: pausing has less user impact than
+    stopping, and paused apps mostly have older last-access times — a single
+    LRU sort across both tiers would systematically stop paused apps instead.
+    Only when nothing is left to pause does the LRU paused app get stopped.
+
+    One demotion per control cycle: the next cycle re-reads PSI and demotes
+    again only if pressure is still high, so a spike frees memory gradually
+    instead of stopping everything at once.
+    """
+    running = []
+    paused = []
+    for app in apps:
+        app_meta = get_app_metadata(app.name)
+        if app_meta.lifecycle.always_on:
+            continue
+        if time.time() - last_access_dict.get(app.name, 0.0) <= RECENT_ACCESS_GRACE:
+            continue
+        if app.status == Status.RUNNING:
+            running.append(app)
+        elif app.status == Status.PAUSED:
+            paused.append(app)
+
+    def lru(candidates: List[InstalledApp]) -> InstalledApp:
+        return min(candidates, key=lambda app: last_access_dict.get(app.name, 0.0))
+
+    if running:
+        victim = lru(running)
+        if get_app_metadata(victim.name).lifecycle.skip_pause:
+            await docker_stop_app(victim.name)
+        else:
+            await docker_pause_app(victim.name)
+    elif paused:
+        await docker_stop_app(lru(paused).name)