Phase 7: Kubernetes learning track on local kind (manifests + HPA + ingress + CI/CD)

.dockerignore

@@ -0,0 +1,17 @@
+# Build-context excludes. The build context for deploy/Dockerfile.* is the REPO ROOT, so without
+# this the host virtualenv (backend/.venv, ~1.3G) and node_modules (frontend, ~660M, wrong-OS
+# native binaries) would be shipped as context AND copied into the images by `COPY backend/ ./`
+# / `COPY frontend/ ./` — bloating the backend image and breaking the frontend image (platform
+# mismatch). Added in Phase 7 (the prod images were only `docker compose config`-linted before).
+**/.venv/
+**/node_modules/
+**/__pycache__/
+**/*.py[cod]
+.git/
+**/.next/
+**/mlruns/
+mlflow.db
+**/.env
+.claude/
+.claude-flow/
+docs/k8s-evidence/

.github/workflows/k8s.yml

@@ -0,0 +1,120 @@
+# Phase 7 — Kubernetes (kind) smoke pipeline (D8).
+# Stands the full stack up on a throwaway multi-node kind cluster, applies the SAME manifests +
+# add-ons used locally, waits for every rollout, smokes /health + the UI through ingress, then
+# tears the cluster down (helm/kind-action deletes it in its post step — D10, nothing left running).
+# This is SEPARATE from the eval-gated ci.yml (which stays untouched). HPA load-scaling is proven
+# locally (D13, docs/k8s-evidence/08) — not asserted here, to keep CI deterministic.
+name: k8s-kind
+
+on:
+  push:
+    paths:
+      - "deploy/**"
+      - ".github/workflows/k8s.yml"
+      - "backend/**"
+      - "frontend/**"
+  pull_request:
+    paths:
+      - "deploy/**"
+      - ".github/workflows/k8s.yml"
+      - "backend/**"
+      - "frontend/**"
+  workflow_dispatch: {}
+
+jobs:
+  kind-smoke:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+
+      # The backend image carries CUDA torch wheels (~several GB); the hosted runner's ~14 GB free
+      # disk overflows when `kind load` does `docker save` to /tmp. Reclaim ~20+ GB of preinstalled
+      # toolchains we don't use (Android SDK, .NET, GHC, CodeQL). (CPU-only torch would slim the
+      # image itself — deferred, see ADR-0014.)
+      - name: Free up runner disk space
+        run: |
+          df -h / | tail -1
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL
+          df -h / | tail -1
+
+      # Multi-node cluster from the committed config (ingress-ready label + host 80/443 maps).
+      # Versions pinned to match local (kind v0.31.0 -> node v1.35.0). Name via flag only.
+      - name: Create kind cluster
+        uses: helm/kind-action@v1
+        with:
+          version: v0.31.0
+          node_image: kindest/node:v1.35.0
+          cluster_name: second-brain
+          config: deploy/k8s/kind-cluster.yaml
+
+      - name: Build images
+        run: |
+          docker build -f deploy/Dockerfile.backend -t second-brain-api:phase7 .
+          docker build -f deploy/Dockerfile.frontend \
+            --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local \
+            -t second-brain-web:phase7 .
+
+      - name: Load images into kind (no registry, D2)
+        run: |
+          kind load docker-image second-brain-api:phase7 --name second-brain
+          kind load docker-image second-brain-web:phase7 --name second-brain
+
+      - name: Namespace + Secret (throwaway CI values) + monitoring ConfigMaps
+        run: |
+          kubectl apply -f deploy/k8s/namespace.yaml
+          kubectl -n second-brain create secret generic second-brain-secrets \
+            --from-literal=POSTGRES_PASSWORD=ci_postgres_pw \
+            --from-literal=SECOND_BRAIN_ADMIN_TOKEN=ci-admin-token \
+            --from-literal=SECOND_BRAIN_GEMINI_API_KEY= \
+            --from-literal=GRAFANA_ADMIN_PASSWORD=ci-admin
+          kubectl -n second-brain create configmap prometheus-config \
+            --from-file=prometheus.yml=deploy/prometheus/prometheus.yml \
+            --from-file=alerts.yml=deploy/prometheus/alerts.yml --dry-run=client -o yaml | kubectl apply -f -
+          kubectl -n second-brain create configmap grafana-datasources \
+            --from-file=deploy/grafana/provisioning/datasources/datasource.yml --dry-run=client -o yaml | kubectl apply -f -
+          kubectl -n second-brain create configmap grafana-dashboard-provider \
+            --from-file=deploy/grafana/provisioning/dashboards/dashboards.yml --dry-run=client -o yaml | kubectl apply -f -
+          kubectl -n second-brain create configmap grafana-dashboard-json \
+            --from-file=deploy/grafana/dashboards/second-brain.json --dry-run=client -o yaml | kubectl apply -f -
+
+      - name: Install ingress-nginx + metrics-server (pinned)
+        run: |
+          kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.3/deploy/static/provider/kind/deploy.yaml
+          kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml
+          kubectl -n kube-system patch deployment metrics-server --type=json \
+            -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]'
+          kubectl wait -n ingress-nginx --for=condition=ready pod \
+            -l app.kubernetes.io/component=controller --timeout=180s
+
+      - name: Apply manifests
+        run: kubectl apply -k deploy/k8s
+
+      - name: Wait for rollouts
+        run: |
+          kubectl -n second-brain rollout status statefulset/db --timeout=300s
+          kubectl -n second-brain wait --for=condition=complete job/migrate --timeout=300s
+          for d in pgbouncer redis api worker frontend prometheus grafana; do
+            kubectl -n second-brain rollout status deploy/$d --timeout=300s
+          done
+
+      - name: Smoke /health + UI through ingress
+        run: |
+          set -euo pipefail
+          ok=""
+          for i in $(seq 1 15); do
+            code=$(curl -s -o /dev/null -w "%{http_code}" -H "Host: api.second-brain.local" http://localhost/health || true)
+            if [ "$code" = "200" ]; then ok="yes"; break; fi
+            echo "attempt $i: api /health -> $code (retrying)"; sleep 5
+          done
+          test "$ok" = "yes" || { echo "api /health never returned 200"; exit 1; }
+          echo "api /health body:"; curl -s -H "Host: api.second-brain.local" http://localhost/health; echo
+          fcode=$(curl -s -L -o /dev/null -w "%{http_code}" -H "Host: second-brain.local" http://localhost/)
+          echo "frontend / (followed) -> $fcode"; test "$fcode" = "200"
+
+      - name: Dump state (always)
+        if: always()
+        run: |
+          kubectl -n second-brain get deploy,statefulset,job,svc,ingress,hpa,pods -o wide || true
+          kubectl -n second-brain get events --sort-by=.lastTimestamp | tail -30 || true
+      # Teardown: helm/kind-action deletes the cluster in its post-job step (D10).

.gitignore

@@ -12,6 +12,8 @@ backend/.env
 # Phase 6 production secrets — only the *.example templates are committed
 deploy/.env.prod
 deploy/pgbouncer/userlist.txt
+# Phase 7 K8s secret — only secret.example.yaml is committed (D4)
+deploy/k8s/secret.yaml
 
 # MLflow local tracking store (Phase 3 eval artifacts — regenerated by app.eval.runner)
 mlruns/

README.md

@@ -60,13 +60,13 @@ and everything else is self-hosted in Docker Compose on a single ~$4–6/mo VPS.
 | Phase | Description | Status |
 |:---:|---|:---:|
 | **0** | Data model · ER diagram · Alembic migrations · pgvector/full-text indexes | ✅ Complete |
-| 1 | RAG MVP — FastAPI `/ingest` + `/chat`, hybrid retrieval, `LLMClient` | ⬜ Next |
-| 2 | Next.js chat UI — streaming, citations, semantic search | ⬜ |
-| 3 | Evaluation + MLOps — eval set, MLflow, A/B, prompt versioning + rollback | ⬜ |
-| 4 | MCP server + agentic actions (incl. self-research) | ⬜ |
-| 5 | Daily briefing + scheduled pipelines | ⬜ |
-| 6 | Productionize on VPS + data-ops hardening (RLS, retention, pooling, tuning) | ⬜ |
-| 7 | Kubernetes learning track on local k3s/kind | ⬜ |
+| 1 | RAG MVP — FastAPI `/ingest` + `/chat`, hybrid retrieval, `LLMClient` | ✅ Complete |
+| 2 | Next.js chat UI — streaming, citations, semantic search | ✅ Complete |
+| 3 | Evaluation + MLOps — eval set, MLflow, A/B, prompt versioning + rollback | ✅ Complete |
+| 4 | MCP server + agentic actions (incl. self-research) | ✅ Complete |
+| 5 | Daily briefing + scheduled pipelines | ✅ Complete |
+| 6 | Productionize on VPS + data-ops hardening (RLS, retention, pooling, tuning) | ✅ Complete |
+| 7 | Kubernetes learning track on local k3s/kind | ✅ Complete |
 
 Live status & dated log: [`docs/PROGRESS.md`](docs/PROGRESS.md).
 
@@ -81,6 +81,9 @@ second-brain/
 │   ├── app/db/               # SQLAlchemy models, settings
 │   ├── migrations/           # Alembic env + versioned migrations
 │   └── README.md             # backend run & verify guide
+├── deploy/                   # prod Docker Compose stack + Phase 7 Kubernetes manifests
+│   ├── docker-compose.prod.yml
+│   └── k8s/                  # kind learning-track manifests + README (run/verify/teardown)
 └── docs/
     ├── project-plan.md       # complete plan + JD-coverage matrix
     ├── PROGRESS.md           # running status log
@@ -113,6 +116,34 @@ docker exec -it second_brain_db psql -U second_brain -d second_brain -c "\dt"
 Full verification steps (HNSW index, generated tsvector column) are in
 [`backend/README.md`](backend/README.md).
 
+## ☸️ Phase 7 — Kubernetes learning track (run & verify)
+
+Kubernetes here is a **learning track, not the production runtime** (prod stays the single-VPS
+Docker Compose stack). The manifests in [`deploy/k8s/`](deploy/k8s/) prove the whole stack runs on
+real K8s — Postgres StatefulSet+PVC, a migrate Job, api/worker/frontend Deployments, ingress-nginx,
+an HPA that scales `api` under load, and Prometheus+Grafana — then the cluster is **torn down** so
+nothing keeps running ($0). Decisions in [ADR-0014](docs/adr/0014-kubernetes-learning-track.md);
+captured evidence in [`docs/k8s-evidence/`](docs/k8s-evidence/); CI in
+[`.github/workflows/k8s.yml`](.github/workflows/k8s.yml).
+
+```bash
+# Requires Docker Desktop + kind + kubectl. Full guide: deploy/k8s/README.md
+kind create cluster --name second-brain --config deploy/k8s/kind-cluster.yaml
+docker build -f deploy/Dockerfile.backend  -t second-brain-api:phase7 .
+docker build -f deploy/Dockerfile.frontend -t second-brain-web:phase7 \
+  --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local .
+kind load docker-image second-brain-api:phase7 --name second-brain
+kind load docker-image second-brain-web:phase7 --name second-brain
+# ... create Secret + monitoring ConfigMaps + install ingress-nginx/metrics-server (see deploy/k8s/README.md) ...
+kubectl apply -k deploy/k8s
+
+# Verify (host 80 maps into the cluster):
+curl -H 'Host: api.second-brain.local' http://localhost/health    # {"status":"ok","db":"ok",...}
+curl -L -H 'Host: second-brain.local'  http://localhost/          # UI (/ -> /chat)
+
+kind delete cluster --name second-brain                           # teardown — leave nothing running
+```
+
 ## 📐 Architecture & Decisions
 
 - **System design & cost model** → [`docs/project-plan.md`](docs/project-plan.md)

deploy/Dockerfile.frontend

@@ -11,6 +11,15 @@ COPY frontend/package.json frontend/package-lock.json* ./
 RUN npm install --no-audit --no-fund
 
 COPY frontend/ ./
+
+# Phase 7 (D11): Next.js inlines NEXT_PUBLIC_* at build time, so the API base URL must be set
+# BEFORE `next build`, not at runtime. The default preserves the prior behaviour (localhost:8000,
+# used by docker-compose which passes no build arg); the K8s image is built with
+# --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local so the browser calls the API
+# through ingress. Additive + backwards-compatible.
+ARG NEXT_PUBLIC_API_BASE_URL=http://localhost:8000
+ENV NEXT_PUBLIC_API_BASE_URL=$NEXT_PUBLIC_API_BASE_URL
+
 RUN npm run build
 
 ENV NODE_ENV=production

deploy/k8s/README.md

@@ -0,0 +1,85 @@
+# Kubernetes learning track (local `kind`) — Phase 7
+
+> **Kubernetes here is a LEARNING TRACK, not the production runtime.** Production stays the
+> single-VPS Docker Compose stack (`deploy/docker-compose.prod.yml`, ADR-0011/0012). These
+> manifests prove the app runs on real K8s (StatefulSet, Job, Deployments, ingress, HPA,
+> monitoring), then the cluster is **torn down** so nothing keeps running ($0). See ADR-0014 and
+> `docs/phase-7-plan.md`. Evidence captured under `docs/k8s-evidence/`. CI: `.github/workflows/k8s.yml`.
+
+The 8 prod-compose services map to: `db` → StatefulSet+PVC, migrations → a Job, `pgbouncer`/`redis`/
+`api`/`worker`/`frontend`/`prometheus`/`grafana` → Deployments, plus an Ingress and an HPA on `api`.
+
+## Prerequisites
+- Docker Desktop (WSL2) running. `kind` + `kubectl` (`winget install Kubernetes.kind`; kubectl ships with Docker Desktop).
+- The in-cluster Postgres is **separate** from any host Postgres (e.g. the dev DB on host :5433).
+
+## 1. Create the cluster (multi-node, ingress-ready)
+```bash
+kind create cluster --name second-brain --config deploy/k8s/kind-cluster.yaml
+```
+
+## 2. Build images and load them into the cluster (no registry, D2)
+```bash
+docker build -f deploy/Dockerfile.backend  -t second-brain-api:phase7 .
+docker build -f deploy/Dockerfile.frontend -t second-brain-web:phase7 \
+  --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local .   # D11: baked at build time
+kind load docker-image second-brain-api:phase7 --name second-brain
+kind load docker-image second-brain-web:phase7 --name second-brain
+```
+
+## 3. Create the Secret (NOT committed, D4) + the monitoring ConfigMaps (from the Phase 6 configs)
+```bash
+kubectl apply -f deploy/k8s/namespace.yaml
+kubectl -n second-brain create secret generic second-brain-secrets \
+  --from-literal=POSTGRES_PASSWORD='second_brain' \
+  --from-literal=SECOND_BRAIN_ADMIN_TOKEN='phase7-admin-token' \
+  --from-literal=SECOND_BRAIN_GEMINI_API_KEY='' \
+  --from-literal=GRAFANA_ADMIN_PASSWORD='admin'
+
+kubectl -n second-brain create configmap prometheus-config \
+  --from-file=prometheus.yml=deploy/prometheus/prometheus.yml \
+  --from-file=alerts.yml=deploy/prometheus/alerts.yml --dry-run=client -o yaml | kubectl apply -f -
+kubectl -n second-brain create configmap grafana-datasources \
+  --from-file=deploy/grafana/provisioning/datasources/datasource.yml --dry-run=client -o yaml | kubectl apply -f -
+kubectl -n second-brain create configmap grafana-dashboard-provider \
+  --from-file=deploy/grafana/provisioning/dashboards/dashboards.yml --dry-run=client -o yaml | kubectl apply -f -
+kubectl -n second-brain create configmap grafana-dashboard-json \
+  --from-file=deploy/grafana/dashboards/second-brain.json --dry-run=client -o yaml | kubectl apply -f -
+```
+
+## 4. Cluster add-ons (pinned)
+```bash
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.3/deploy/static/provider/kind/deploy.yaml
+kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml
+kubectl -n kube-system patch deployment metrics-server --type=json \
+  -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]'   # kind needs this
+kubectl wait -n ingress-nginx --for=condition=ready pod -l app.kubernetes.io/component=controller --timeout=180s
+```
+
+## 5. Apply the stack
+```bash
+kubectl apply -k deploy/k8s     # one-shot (Secret + monitoring ConfigMaps from step 3 are prerequisites)
+# Wait for everything:
+kubectl -n second-brain rollout status statefulset/db
+kubectl -n second-brain wait --for=condition=complete job/migrate --timeout=300s
+for d in pgbouncer redis api worker frontend prometheus grafana; do kubectl -n second-brain rollout status deploy/$d; done
+```
+
+## 6. Verify (smoke through ingress — host 80 maps to the cluster)
+```bash
+curl -H 'Host: api.second-brain.local' http://localhost/health        # {"status":"ok","db":"ok",...}
+curl -L -H 'Host: second-brain.local'  http://localhost/              # UI (/, 307 -> /chat, 200 HTML)
+```
+For a browser, add to your hosts file: `127.0.0.1  second-brain.local api.second-brain.local`.
+
+## 7. HPA autoscaling demo (D6)
+```bash
+kubectl -n second-brain run load --image=williamyeh/hey --restart=Never -- -z 90s -c 80 http://api:8000/health
+watch kubectl -n second-brain get hpa api          # CPU climbs past 50%; api scales 1 -> 4
+kubectl -n second-brain delete pod load --now      # then api scales 4 -> 1
+```
+
+## 8. Teardown (D10 — leave nothing running, $0)
+```bash
+kind delete cluster --name second-brain
+```

deploy/k8s/api-hpa.yaml

@@ -0,0 +1,39 @@
+# Horizontal Pod Autoscaler for the api (D6). Scales on CPU utilisation vs the api's CPU `requests`
+# (250m). averageUtilization 50% means sustained load >125m/pod triggers scale-up. Requires
+# metrics-server (installed in Task 8). Demonstrated by driving load at /health and watching
+# replicas climb 1 -> N (evidence captured under docs/k8s-evidence/).
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: api
+  namespace: second-brain
+  labels:
+    app.kubernetes.io/part-of: second-brain
+    app: api
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: api
+  minReplicas: 1
+  maxReplicas: 4
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 50
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 0        # react fast for the demo
+      policies:
+        - type: Pods
+          value: 2
+          periodSeconds: 15
+    scaleDown:
+      stabilizationWindowSeconds: 60
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 30