diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..1586d86 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Build-context excludes. The build context for deploy/Dockerfile.* is the REPO ROOT, so without +# this the host virtualenv (backend/.venv, ~1.3G) and node_modules (frontend, ~660M, wrong-OS +# native binaries) would be shipped as context AND copied into the images by `COPY backend/ ./` +# / `COPY frontend/ ./` — bloating the backend image and breaking the frontend image (platform +# mismatch). Added in Phase 7 (the prod images were only `docker compose config`-linted before). +**/.venv/ +**/node_modules/ +**/__pycache__/ +**/*.py[cod] +.git/ +**/.next/ +**/mlruns/ +mlflow.db +**/.env +.claude/ +.claude-flow/ +docs/k8s-evidence/ diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml new file mode 100644 index 0000000..e033bcb --- /dev/null +++ b/.github/workflows/k8s.yml @@ -0,0 +1,120 @@ +# Phase 7 — Kubernetes (kind) smoke pipeline (D8). +# Stands the full stack up on a throwaway multi-node kind cluster, applies the SAME manifests + +# add-ons used locally, waits for every rollout, smokes /health + the UI through ingress, then +# tears the cluster down (helm/kind-action deletes it in its post step — D10, nothing left running). +# This is SEPARATE from the eval-gated ci.yml (which stays untouched). HPA load-scaling is proven +# locally (D13, docs/k8s-evidence/08) — not asserted here, to keep CI deterministic. +name: k8s-kind + +on: + push: + paths: + - "deploy/**" + - ".github/workflows/k8s.yml" + - "backend/**" + - "frontend/**" + pull_request: + paths: + - "deploy/**" + - ".github/workflows/k8s.yml" + - "backend/**" + - "frontend/**" + workflow_dispatch: {} + +jobs: + kind-smoke: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + # The backend image carries CUDA torch wheels (~several GB); the hosted runner's ~14 GB free + # disk overflows when `kind load` does `docker save` to /tmp. Reclaim ~20+ GB of preinstalled + # toolchains we don't use (Android SDK, .NET, GHC, CodeQL). (CPU-only torch would slim the + # image itself — deferred, see ADR-0014.) + - name: Free up runner disk space + run: | + df -h / | tail -1 + sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL + df -h / | tail -1 + + # Multi-node cluster from the committed config (ingress-ready label + host 80/443 maps). + # Versions pinned to match local (kind v0.31.0 -> node v1.35.0). Name via flag only. + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + version: v0.31.0 + node_image: kindest/node:v1.35.0 + cluster_name: second-brain + config: deploy/k8s/kind-cluster.yaml + + - name: Build images + run: | + docker build -f deploy/Dockerfile.backend -t second-brain-api:phase7 . + docker build -f deploy/Dockerfile.frontend \ + --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local \ + -t second-brain-web:phase7 . + + - name: Load images into kind (no registry, D2) + run: | + kind load docker-image second-brain-api:phase7 --name second-brain + kind load docker-image second-brain-web:phase7 --name second-brain + + - name: Namespace + Secret (throwaway CI values) + monitoring ConfigMaps + run: | + kubectl apply -f deploy/k8s/namespace.yaml + kubectl -n second-brain create secret generic second-brain-secrets \ + --from-literal=POSTGRES_PASSWORD=ci_postgres_pw \ + --from-literal=SECOND_BRAIN_ADMIN_TOKEN=ci-admin-token \ + --from-literal=SECOND_BRAIN_GEMINI_API_KEY= \ + --from-literal=GRAFANA_ADMIN_PASSWORD=ci-admin + kubectl -n second-brain create configmap prometheus-config \ + --from-file=prometheus.yml=deploy/prometheus/prometheus.yml \ + --from-file=alerts.yml=deploy/prometheus/alerts.yml --dry-run=client -o yaml | kubectl apply -f - + kubectl -n second-brain create configmap grafana-datasources \ + --from-file=deploy/grafana/provisioning/datasources/datasource.yml --dry-run=client -o yaml | kubectl apply -f - + kubectl -n second-brain create configmap grafana-dashboard-provider \ + --from-file=deploy/grafana/provisioning/dashboards/dashboards.yml --dry-run=client -o yaml | kubectl apply -f - + kubectl -n second-brain create configmap grafana-dashboard-json \ + --from-file=deploy/grafana/dashboards/second-brain.json --dry-run=client -o yaml | kubectl apply -f - + + - name: Install ingress-nginx + metrics-server (pinned) + run: | + kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.3/deploy/static/provider/kind/deploy.yaml + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml + kubectl -n kube-system patch deployment metrics-server --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' + kubectl wait -n ingress-nginx --for=condition=ready pod \ + -l app.kubernetes.io/component=controller --timeout=180s + + - name: Apply manifests + run: kubectl apply -k deploy/k8s + + - name: Wait for rollouts + run: | + kubectl -n second-brain rollout status statefulset/db --timeout=300s + kubectl -n second-brain wait --for=condition=complete job/migrate --timeout=300s + for d in pgbouncer redis api worker frontend prometheus grafana; do + kubectl -n second-brain rollout status deploy/$d --timeout=300s + done + + - name: Smoke /health + UI through ingress + run: | + set -euo pipefail + ok="" + for i in $(seq 1 15); do + code=$(curl -s -o /dev/null -w "%{http_code}" -H "Host: api.second-brain.local" http://localhost/health || true) + if [ "$code" = "200" ]; then ok="yes"; break; fi + echo "attempt $i: api /health -> $code (retrying)"; sleep 5 + done + test "$ok" = "yes" || { echo "api /health never returned 200"; exit 1; } + echo "api /health body:"; curl -s -H "Host: api.second-brain.local" http://localhost/health; echo + fcode=$(curl -s -L -o /dev/null -w "%{http_code}" -H "Host: second-brain.local" http://localhost/) + echo "frontend / (followed) -> $fcode"; test "$fcode" = "200" + + - name: Dump state (always) + if: always() + run: | + kubectl -n second-brain get deploy,statefulset,job,svc,ingress,hpa,pods -o wide || true + kubectl -n second-brain get events --sort-by=.lastTimestamp | tail -30 || true + # Teardown: helm/kind-action deletes the cluster in its post-job step (D10). diff --git a/.gitignore b/.gitignore index af729ea..4bb857d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ backend/.env # Phase 6 production secrets — only the *.example templates are committed deploy/.env.prod deploy/pgbouncer/userlist.txt +# Phase 7 K8s secret — only secret.example.yaml is committed (D4) +deploy/k8s/secret.yaml # MLflow local tracking store (Phase 3 eval artifacts — regenerated by app.eval.runner) mlruns/ diff --git a/README.md b/README.md index 83face3..b9be911 100644 --- a/README.md +++ b/README.md @@ -60,13 +60,13 @@ and everything else is self-hosted in Docker Compose on a single ~$4–6/mo VPS. | Phase | Description | Status | |:---:|---|:---:| | **0** | Data model · ER diagram · Alembic migrations · pgvector/full-text indexes | ✅ Complete | -| 1 | RAG MVP — FastAPI `/ingest` + `/chat`, hybrid retrieval, `LLMClient` | ⬜ Next | -| 2 | Next.js chat UI — streaming, citations, semantic search | ⬜ | -| 3 | Evaluation + MLOps — eval set, MLflow, A/B, prompt versioning + rollback | ⬜ | -| 4 | MCP server + agentic actions (incl. self-research) | ⬜ | -| 5 | Daily briefing + scheduled pipelines | ⬜ | -| 6 | Productionize on VPS + data-ops hardening (RLS, retention, pooling, tuning) | ⬜ | -| 7 | Kubernetes learning track on local k3s/kind | ⬜ | +| 1 | RAG MVP — FastAPI `/ingest` + `/chat`, hybrid retrieval, `LLMClient` | ✅ Complete | +| 2 | Next.js chat UI — streaming, citations, semantic search | ✅ Complete | +| 3 | Evaluation + MLOps — eval set, MLflow, A/B, prompt versioning + rollback | ✅ Complete | +| 4 | MCP server + agentic actions (incl. self-research) | ✅ Complete | +| 5 | Daily briefing + scheduled pipelines | ✅ Complete | +| 6 | Productionize on VPS + data-ops hardening (RLS, retention, pooling, tuning) | ✅ Complete | +| 7 | Kubernetes learning track on local k3s/kind | ✅ Complete | Live status & dated log: [`docs/PROGRESS.md`](docs/PROGRESS.md). @@ -81,6 +81,9 @@ second-brain/ │ ├── app/db/ # SQLAlchemy models, settings │ ├── migrations/ # Alembic env + versioned migrations │ └── README.md # backend run & verify guide +├── deploy/ # prod Docker Compose stack + Phase 7 Kubernetes manifests +│ ├── docker-compose.prod.yml +│ └── k8s/ # kind learning-track manifests + README (run/verify/teardown) └── docs/ ├── project-plan.md # complete plan + JD-coverage matrix ├── PROGRESS.md # running status log @@ -113,6 +116,34 @@ docker exec -it second_brain_db psql -U second_brain -d second_brain -c "\dt" Full verification steps (HNSW index, generated tsvector column) are in [`backend/README.md`](backend/README.md). +## ☸️ Phase 7 — Kubernetes learning track (run & verify) + +Kubernetes here is a **learning track, not the production runtime** (prod stays the single-VPS +Docker Compose stack). The manifests in [`deploy/k8s/`](deploy/k8s/) prove the whole stack runs on +real K8s — Postgres StatefulSet+PVC, a migrate Job, api/worker/frontend Deployments, ingress-nginx, +an HPA that scales `api` under load, and Prometheus+Grafana — then the cluster is **torn down** so +nothing keeps running ($0). Decisions in [ADR-0014](docs/adr/0014-kubernetes-learning-track.md); +captured evidence in [`docs/k8s-evidence/`](docs/k8s-evidence/); CI in +[`.github/workflows/k8s.yml`](.github/workflows/k8s.yml). + +```bash +# Requires Docker Desktop + kind + kubectl. Full guide: deploy/k8s/README.md +kind create cluster --name second-brain --config deploy/k8s/kind-cluster.yaml +docker build -f deploy/Dockerfile.backend -t second-brain-api:phase7 . +docker build -f deploy/Dockerfile.frontend -t second-brain-web:phase7 \ + --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local . +kind load docker-image second-brain-api:phase7 --name second-brain +kind load docker-image second-brain-web:phase7 --name second-brain +# ... create Secret + monitoring ConfigMaps + install ingress-nginx/metrics-server (see deploy/k8s/README.md) ... +kubectl apply -k deploy/k8s + +# Verify (host 80 maps into the cluster): +curl -H 'Host: api.second-brain.local' http://localhost/health # {"status":"ok","db":"ok",...} +curl -L -H 'Host: second-brain.local' http://localhost/ # UI (/ -> /chat) + +kind delete cluster --name second-brain # teardown — leave nothing running +``` + ## 📐 Architecture & Decisions - **System design & cost model** → [`docs/project-plan.md`](docs/project-plan.md) diff --git a/deploy/Dockerfile.frontend b/deploy/Dockerfile.frontend index 04fd6db..b26bb72 100644 --- a/deploy/Dockerfile.frontend +++ b/deploy/Dockerfile.frontend @@ -11,6 +11,15 @@ COPY frontend/package.json frontend/package-lock.json* ./ RUN npm install --no-audit --no-fund COPY frontend/ ./ + +# Phase 7 (D11): Next.js inlines NEXT_PUBLIC_* at build time, so the API base URL must be set +# BEFORE `next build`, not at runtime. The default preserves the prior behaviour (localhost:8000, +# used by docker-compose which passes no build arg); the K8s image is built with +# --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local so the browser calls the API +# through ingress. Additive + backwards-compatible. +ARG NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 +ENV NEXT_PUBLIC_API_BASE_URL=$NEXT_PUBLIC_API_BASE_URL + RUN npm run build ENV NODE_ENV=production diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md new file mode 100644 index 0000000..a9d7ac3 --- /dev/null +++ b/deploy/k8s/README.md @@ -0,0 +1,85 @@ +# Kubernetes learning track (local `kind`) — Phase 7 + +> **Kubernetes here is a LEARNING TRACK, not the production runtime.** Production stays the +> single-VPS Docker Compose stack (`deploy/docker-compose.prod.yml`, ADR-0011/0012). These +> manifests prove the app runs on real K8s (StatefulSet, Job, Deployments, ingress, HPA, +> monitoring), then the cluster is **torn down** so nothing keeps running ($0). See ADR-0014 and +> `docs/phase-7-plan.md`. Evidence captured under `docs/k8s-evidence/`. CI: `.github/workflows/k8s.yml`. + +The 8 prod-compose services map to: `db` → StatefulSet+PVC, migrations → a Job, `pgbouncer`/`redis`/ +`api`/`worker`/`frontend`/`prometheus`/`grafana` → Deployments, plus an Ingress and an HPA on `api`. + +## Prerequisites +- Docker Desktop (WSL2) running. `kind` + `kubectl` (`winget install Kubernetes.kind`; kubectl ships with Docker Desktop). +- The in-cluster Postgres is **separate** from any host Postgres (e.g. the dev DB on host :5433). + +## 1. Create the cluster (multi-node, ingress-ready) +```bash +kind create cluster --name second-brain --config deploy/k8s/kind-cluster.yaml +``` + +## 2. Build images and load them into the cluster (no registry, D2) +```bash +docker build -f deploy/Dockerfile.backend -t second-brain-api:phase7 . +docker build -f deploy/Dockerfile.frontend -t second-brain-web:phase7 \ + --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local . # D11: baked at build time +kind load docker-image second-brain-api:phase7 --name second-brain +kind load docker-image second-brain-web:phase7 --name second-brain +``` + +## 3. Create the Secret (NOT committed, D4) + the monitoring ConfigMaps (from the Phase 6 configs) +```bash +kubectl apply -f deploy/k8s/namespace.yaml +kubectl -n second-brain create secret generic second-brain-secrets \ + --from-literal=POSTGRES_PASSWORD='second_brain' \ + --from-literal=SECOND_BRAIN_ADMIN_TOKEN='phase7-admin-token' \ + --from-literal=SECOND_BRAIN_GEMINI_API_KEY='' \ + --from-literal=GRAFANA_ADMIN_PASSWORD='admin' + +kubectl -n second-brain create configmap prometheus-config \ + --from-file=prometheus.yml=deploy/prometheus/prometheus.yml \ + --from-file=alerts.yml=deploy/prometheus/alerts.yml --dry-run=client -o yaml | kubectl apply -f - +kubectl -n second-brain create configmap grafana-datasources \ + --from-file=deploy/grafana/provisioning/datasources/datasource.yml --dry-run=client -o yaml | kubectl apply -f - +kubectl -n second-brain create configmap grafana-dashboard-provider \ + --from-file=deploy/grafana/provisioning/dashboards/dashboards.yml --dry-run=client -o yaml | kubectl apply -f - +kubectl -n second-brain create configmap grafana-dashboard-json \ + --from-file=deploy/grafana/dashboards/second-brain.json --dry-run=client -o yaml | kubectl apply -f - +``` + +## 4. Cluster add-ons (pinned) +```bash +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.3/deploy/static/provider/kind/deploy.yaml +kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml +kubectl -n kube-system patch deployment metrics-server --type=json \ + -p '[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-insecure-tls"}]' # kind needs this +kubectl wait -n ingress-nginx --for=condition=ready pod -l app.kubernetes.io/component=controller --timeout=180s +``` + +## 5. Apply the stack +```bash +kubectl apply -k deploy/k8s # one-shot (Secret + monitoring ConfigMaps from step 3 are prerequisites) +# Wait for everything: +kubectl -n second-brain rollout status statefulset/db +kubectl -n second-brain wait --for=condition=complete job/migrate --timeout=300s +for d in pgbouncer redis api worker frontend prometheus grafana; do kubectl -n second-brain rollout status deploy/$d; done +``` + +## 6. Verify (smoke through ingress — host 80 maps to the cluster) +```bash +curl -H 'Host: api.second-brain.local' http://localhost/health # {"status":"ok","db":"ok",...} +curl -L -H 'Host: second-brain.local' http://localhost/ # UI (/, 307 -> /chat, 200 HTML) +``` +For a browser, add to your hosts file: `127.0.0.1 second-brain.local api.second-brain.local`. + +## 7. HPA autoscaling demo (D6) +```bash +kubectl -n second-brain run load --image=williamyeh/hey --restart=Never -- -z 90s -c 80 http://api:8000/health +watch kubectl -n second-brain get hpa api # CPU climbs past 50%; api scales 1 -> 4 +kubectl -n second-brain delete pod load --now # then api scales 4 -> 1 +``` + +## 8. Teardown (D10 — leave nothing running, $0) +```bash +kind delete cluster --name second-brain +``` diff --git a/deploy/k8s/api-hpa.yaml b/deploy/k8s/api-hpa.yaml new file mode 100644 index 0000000..7f07694 --- /dev/null +++ b/deploy/k8s/api-hpa.yaml @@ -0,0 +1,39 @@ +# Horizontal Pod Autoscaler for the api (D6). Scales on CPU utilisation vs the api's CPU `requests` +# (250m). averageUtilization 50% means sustained load >125m/pod triggers scale-up. Requires +# metrics-server (installed in Task 8). Demonstrated by driving load at /health and watching +# replicas climb 1 -> N (evidence captured under docs/k8s-evidence/). +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: api + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: api +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: api + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 # react fast for the demo + policies: + - type: Pods + value: 2 + periodSeconds: 15 + scaleDown: + stabilizationWindowSeconds: 60 + policies: + - type: Pods + value: 1 + periodSeconds: 30 diff --git a/deploy/k8s/api.yaml b/deploy/k8s/api.yaml new file mode 100644 index 0000000..5c08594 --- /dev/null +++ b/deploy/k8s/api.yaml @@ -0,0 +1,104 @@ +# FastAPI backend. Runs ONLY uvicorn (the image's default CMD) — migrations are the migrate Job's +# job (D3), so this never runs alembic. Connects to Postgres via pgbouncer:6432. CPU `requests` +# are set so the HPA (api-hpa.yaml) can compute a CPU-utilisation %. The embedder (MiniLM/torch) +# loads lazily on first ingest/chat, NOT at startup, so /health stays cheap. +apiVersion: v1 +kind: Service +metadata: + name: api + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: api +spec: + selector: + app: api + ports: + - name: http + port: 8000 + targetPort: 8000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: api +spec: + replicas: 1 # HPA owns the replica count from here (minReplicas: 1) + selector: + matchLabels: + app: api + template: + metadata: + labels: + app: api + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: api + image: second-brain-api:phase7 + imagePullPolicy: IfNotPresent # use the kind-loaded image; never pull from a registry (D2) + ports: + - name: http + containerPort: 8000 + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_DB + # App traffic goes through PgBouncer (session pooling). Assembled via $(VAR) so the + # password lives only in the Secret. + - name: SECOND_BRAIN_DATABASE_URL + value: postgresql+psycopg://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@pgbouncer:6432/$(POSTGRES_DB) + - name: SECOND_BRAIN_LLM_PROVIDER + valueFrom: + configMapKeyRef: + name: second-brain-config + key: SECOND_BRAIN_LLM_PROVIDER + - name: SECOND_BRAIN_GEMINI_API_KEY + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: SECOND_BRAIN_GEMINI_API_KEY + - name: SECOND_BRAIN_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: SECOND_BRAIN_ADMIN_TOKEN + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 5 + failureThreshold: 6 + resources: + requests: + cpu: "250m" # HPA target is a % of this + memory: "512Mi" + limits: + cpu: "1000m" + memory: "1Gi" diff --git a/deploy/k8s/configmap.yaml b/deploy/k8s/configmap.yaml new file mode 100644 index 0000000..dcd7bd0 --- /dev/null +++ b/deploy/k8s/configmap.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: second-brain-config + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain +data: + # Non-secret Postgres identity. The password lives in the Secret; each pod assembles its DSN + # from these + POSTGRES_PASSWORD via $(VAR) substitution (see api/worker/migrate manifests), + # so the credential is never written into a ConfigMap. + POSTGRES_USER: second_brain + POSTGRES_DB: second_brain + # LLM driver (D4): the learning-track demo uses the deterministic, keyless `fake` provider, so + # the whole stack runs $0 with no network/secret and CI stays keyless. Flip to `gemini` and set + # SECOND_BRAIN_GEMINI_API_KEY in the Secret for real answers. + SECOND_BRAIN_LLM_PROVIDER: fake +# NOTE (D11): NEXT_PUBLIC_API_BASE_URL is deliberately NOT here. Next.js inlines NEXT_PUBLIC_* at +# `next build`, so it is BAKED INTO the frontend image (deploy/Dockerfile.frontend ARG), not read +# at runtime. A runtime ConfigMap value would be silently ignored by the browser bundle. diff --git a/deploy/k8s/frontend.yaml b/deploy/k8s/frontend.yaml new file mode 100644 index 0000000..f0a70b3 --- /dev/null +++ b/deploy/k8s/frontend.yaml @@ -0,0 +1,65 @@ +# Next.js frontend. The image was built with NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local +# baked in (D11) — Next inlines NEXT_PUBLIC_* at build time, so the browser calls the API through +# ingress. `npm run start` serves on :3000 (the image's default CMD). +apiVersion: v1 +kind: Service +metadata: + name: frontend + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: frontend +spec: + selector: + app: frontend + ports: + - name: http + port: 3000 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: frontend +spec: + replicas: 1 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: frontend + image: second-brain-web:phase7 + imagePullPolicy: IfNotPresent # use the kind-loaded image; never pull from a registry (D2) + ports: + - name: http + containerPort: 3000 + readinessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 12 + livenessProbe: + httpGet: + path: / + port: 3000 + initialDelaySeconds: 20 + periodSeconds: 15 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/deploy/k8s/ingress.yaml b/deploy/k8s/ingress.yaml new file mode 100644 index 0000000..70079ea --- /dev/null +++ b/deploy/k8s/ingress.yaml @@ -0,0 +1,38 @@ +# North-south ingress (D5) via ingress-nginx. Host-based routing: +# http://second-brain.local -> frontend Service :3000 (the UI) +# http://api.second-brain.local -> api Service :8000 (the API; /health smoked through it) +# +# On the kind cluster the ingress-nginx controller listens on the control-plane node's host ports +# 80/443 (see kind-cluster.yaml extraPortMappings), so these hosts resolve at http://localhost. +# For a browser, add to your hosts file: 127.0.0.1 second-brain.local api.second-brain.local +# For curl smoke tests no hosts edit is needed: curl -H 'Host: api.second-brain.local' http://localhost/health +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: second-brain + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain +spec: + ingressClassName: nginx + rules: + - host: api.second-brain.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: api + port: + number: 8000 + - host: second-brain.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend + port: + number: 3000 diff --git a/deploy/k8s/kind-cluster.yaml b/deploy/k8s/kind-cluster.yaml new file mode 100644 index 0000000..2e82f37 --- /dev/null +++ b/deploy/k8s/kind-cluster.yaml @@ -0,0 +1,36 @@ +# Multi-node kind cluster for the Phase 7 Kubernetes LEARNING TRACK (D1). +# +# Multi-node (1 control-plane + 2 workers) so HPA pod-spread and ingress-on-a-labelled-node +# are real, not single-node toys. The control-plane node is labelled `ingress-ready=true` and +# forwards host ports 80/443 to itself, so the ingress-nginx *kind* provider (which schedules on +# that node with hostPort 80/443) is reachable from the host at http://localhost. +# +# Create: kind create cluster --name second-brain --config deploy/k8s/kind-cluster.yaml +# Teardown: kind delete cluster --name second-brain # D10 — nothing left running ($0) +# +# The cluster name is supplied via the --name flag (NOT a `name:` field here) so it is a single +# source of truth that both the local command and the CI kind-action set the same way — kind +# rejects a name given in both the config and the flag. +# +# Node image follows the installed kind version's default (kind v0.31.0 -> Kubernetes v1.35). +# The manifests use only stable APIs (apps/v1, batch/v1, autoscaling/v2, networking.k8s.io/v1), +# so they are version-robust; CI pins the kind version to keep local and CI aligned. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + - role: worker + - role: worker diff --git a/deploy/k8s/kustomization.yaml b/deploy/k8s/kustomization.yaml new file mode 100644 index 0000000..aea5f0e --- /dev/null +++ b/deploy/k8s/kustomization.yaml @@ -0,0 +1,36 @@ +# Convenience one-shot apply of the core stack: kubectl apply -k deploy/k8s +# +# NOTE: kustomize applies all resources together; it does NOT order them. Readiness converges via +# probes + the migrate Job's OnFailure/backoff (api/worker tolerate a brief pre-migration race). +# +# PREREQUISITES (not in this kustomization, by design): +# 1. The Secret (D4 — created out-of-band; see secret.example.yaml). +# 2. The monitoring ConfigMaps, sourced --from-file from the Phase 6 configs (DRY — kustomize's +# configMapGenerator can't read files above deploy/k8s, so these stay an explicit step): +# kubectl -n second-brain create configmap prometheus-config \ +# --from-file=prometheus.yml=deploy/prometheus/prometheus.yml \ +# --from-file=alerts.yml=deploy/prometheus/alerts.yml --dry-run=client -o yaml | kubectl apply -f - +# kubectl -n second-brain create configmap grafana-datasources --from-file=deploy/grafana/provisioning/datasources/datasource.yml --dry-run=client -o yaml | kubectl apply -f - +# kubectl -n second-brain create configmap grafana-dashboard-provider --from-file=deploy/grafana/provisioning/dashboards/dashboards.yml --dry-run=client -o yaml | kubectl apply -f - +# kubectl -n second-brain create configmap grafana-dashboard-json --from-file=deploy/grafana/dashboards/second-brain.json --dry-run=client -o yaml | kubectl apply -f - +# 3. ingress-nginx + metrics-server (cluster add-ons; see README.md / k8s.yml). +# +# For the layered "apply + wait per layer" path used during development, see README.md. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: second-brain + +resources: + - namespace.yaml + - configmap.yaml + - postgres-statefulset.yaml + - migrate-job.yaml + - pgbouncer.yaml + - redis.yaml + - api.yaml + - worker.yaml + - frontend.yaml + - ingress.yaml + - api-hpa.yaml + - monitoring/prometheus.yaml + - monitoring/grafana.yaml diff --git a/deploy/k8s/migrate-job.yaml b/deploy/k8s/migrate-job.yaml new file mode 100644 index 0000000..77e7ad3 --- /dev/null +++ b/deploy/k8s/migrate-job.yaml @@ -0,0 +1,54 @@ +# One-shot migration Job (D3): `alembic upgrade head` against Postgres DIRECTLY (db:5432, not via +# pgbouncer) — mirrors the compose DIRECT_DATABASE_URL. In compose the api `command` prefixed the +# migration; here it is its own Job so the api/worker run only their process. Re-running is safe +# (alembic is idempotent — already-applied revisions are skipped). Apply AFTER Postgres is Ready; +# OnFailure + backoffLimit retries cover a brief DB-not-ready race. +apiVersion: batch/v1 +kind: Job +metadata: + name: migrate + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: migrate +spec: + backoffLimit: 6 + template: + metadata: + labels: + app: migrate + app.kubernetes.io/part-of: second-brain + spec: + restartPolicy: OnFailure + containers: + - name: alembic + image: second-brain-api:phase7 + imagePullPolicy: IfNotPresent # use the kind-loaded image; never pull from a registry (D2) + command: ["alembic", "upgrade", "head"] + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_DB + # alembic reads settings.database_url = SECOND_BRAIN_DATABASE_URL. Assembled from the + # parts above via $(VAR) so the password stays only in the Secret. DIRECT to db:5432. + - name: SECOND_BRAIN_DATABASE_URL + value: postgresql+psycopg://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@db:5432/$(POSTGRES_DB) + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/deploy/k8s/monitoring/grafana.yaml b/deploy/k8s/monitoring/grafana.yaml new file mode 100644 index 0000000..d96639b --- /dev/null +++ b/deploy/k8s/monitoring/grafana.yaml @@ -0,0 +1,98 @@ +# Grafana (D7) — reuses the Phase 6 provisioning + dashboard verbatim (deploy/grafana/*), which +# already points the datasource at the in-cluster `prometheus:9090`. Three ConfigMaps supply the +# files (created --from-file — DRY, no duplicated copies): +# +# kubectl -n second-brain create configmap grafana-datasources \ +# --from-file=deploy/grafana/provisioning/datasources/datasource.yml --dry-run=client -o yaml | kubectl apply -f - +# kubectl -n second-brain create configmap grafana-dashboard-provider \ +# --from-file=deploy/grafana/provisioning/dashboards/dashboards.yml --dry-run=client -o yaml | kubectl apply -f - +# kubectl -n second-brain create configmap grafana-dashboard-json \ +# --from-file=deploy/grafana/dashboards/second-brain.json --dry-run=client -o yaml | kubectl apply -f - +# +# Admin password comes from the Secret. Storage is emptyDir (ephemeral — learning track). +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: grafana +spec: + selector: + app: grafana + ports: + - name: http + port: 3000 + targetPort: 3000 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: grafana +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: grafana + image: grafana/grafana:latest + ports: + - name: http + containerPort: 3000 + env: + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: GRAFANA_ADMIN_PASSWORD + - name: GF_USERS_ALLOW_SIGN_UP + value: "false" + readinessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 15 + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "384Mi" + volumeMounts: + - name: datasources + mountPath: /etc/grafana/provisioning/datasources + - name: dashboard-provider + mountPath: /etc/grafana/provisioning/dashboards + - name: dashboard-json + mountPath: /var/lib/grafana/dashboards + volumes: + - name: datasources + configMap: + name: grafana-datasources + - name: dashboard-provider + configMap: + name: grafana-dashboard-provider + - name: dashboard-json + configMap: + name: grafana-dashboard-json diff --git a/deploy/k8s/monitoring/prometheus.yaml b/deploy/k8s/monitoring/prometheus.yaml new file mode 100644 index 0000000..9c53052 --- /dev/null +++ b/deploy/k8s/monitoring/prometheus.yaml @@ -0,0 +1,84 @@ +# Prometheus (D7) — reuses the Phase 6 scrape + alert config verbatim (deploy/prometheus/*), which +# already targets K8s-resolvable names (`api:8000`). The config is supplied by a ConfigMap created +# --from-file (DRY — single source of truth, no duplicated copy that could drift): +# +# kubectl -n second-brain create configmap prometheus-config \ +# --from-file=prometheus.yml=deploy/prometheus/prometheus.yml \ +# --from-file=alerts.yml=deploy/prometheus/alerts.yml \ +# --dry-run=client -o yaml | kubectl apply -f - +# +# TSDB is an emptyDir (ephemeral — fine for the learning track). +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: prometheus +spec: + selector: + app: prometheus + ports: + - name: http + port: 9090 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: prometheus + image: prom/prometheus:latest + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + ports: + - name: http + containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + initialDelaySeconds: 15 + periodSeconds: 15 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" + volumeMounts: + - name: config + mountPath: /etc/prometheus + - name: data + mountPath: /prometheus + volumes: + - name: config + configMap: + name: prometheus-config + - name: data + emptyDir: {} diff --git a/deploy/k8s/namespace.yaml b/deploy/k8s/namespace.yaml new file mode 100644 index 0000000..0ba2ee3 --- /dev/null +++ b/deploy/k8s/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: second-brain + labels: + app.kubernetes.io/part-of: second-brain + track: phase-7-learning diff --git a/deploy/k8s/pgbouncer.yaml b/deploy/k8s/pgbouncer.yaml new file mode 100644 index 0000000..1b91c00 --- /dev/null +++ b/deploy/k8s/pgbouncer.yaml @@ -0,0 +1,94 @@ +# PgBouncer connection pooler (D12). Configured entirely by ENV (edoburu/pgbouncer auto-generates +# pgbouncer.ini + userlist.txt at start), so the DB password comes from the Secret and is NEVER +# written into a committed userlist.txt (the compose stack mounts one; we don't, to keep the +# credential out of git). SESSION pool mode preserves psycopg3 prepared statements (ADR-0012). +# The api/worker connect to `pgbouncer:6432`; pgbouncer connects upstream to `db:5432`. +apiVersion: v1 +kind: Service +metadata: + name: pgbouncer + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: pgbouncer +spec: + selector: + app: pgbouncer + ports: + - name: pgbouncer + port: 6432 + targetPort: 6432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pgbouncer + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: pgbouncer +spec: + replicas: 1 + selector: + matchLabels: + app: pgbouncer + template: + metadata: + labels: + app: pgbouncer + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: pgbouncer + image: edoburu/pgbouncer:latest + ports: + - name: pgbouncer + containerPort: 6432 + env: + - name: DB_HOST + value: db + - name: DB_PORT + value: "5432" + - name: DB_USER + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_USER + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: POSTGRES_PASSWORD + - name: DB_NAME + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_DB + - name: POOL_MODE + value: session # ADR-0012: session mode keeps psycopg3 prepared statements working + - name: AUTH_TYPE + value: scram-sha-256 # matches Postgres 16 default password encryption + - name: MAX_CLIENT_CONN + value: "100" + - name: DEFAULT_POOL_SIZE + value: "20" + - name: LISTEN_PORT + value: "6432" + readinessProbe: + tcpSocket: + port: 6432 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 12 + livenessProbe: + tcpSocket: + port: 6432 + initialDelaySeconds: 15 + periodSeconds: 10 + resources: + requests: + cpu: "50m" + memory: "32Mi" + limits: + cpu: "200m" + memory: "128Mi" diff --git a/deploy/k8s/postgres-statefulset.yaml b/deploy/k8s/postgres-statefulset.yaml new file mode 100644 index 0000000..22301be --- /dev/null +++ b/deploy/k8s/postgres-statefulset.yaml @@ -0,0 +1,98 @@ +# Postgres (pgvector) as a StatefulSet + PVC (D3). Reached in-cluster by Service DNS: +# db.second-brain.svc.cluster.local (short: db). Headless Service gives the stable pod DNS a +# StatefulSet wants. The PVC (kind's default local-path StorageClass) persists data across pod +# restarts. The dev Postgres on the host (port 5433) is SEPARATE from this in-cluster instance. +apiVersion: v1 +kind: Service +metadata: + name: db + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: db +spec: + clusterIP: None # headless — stable per-pod DNS for the StatefulSet + selector: + app: db + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: db + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: db +spec: + serviceName: db + replicas: 1 + selector: + matchLabels: + app: db + template: + metadata: + labels: + app: db + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: postgres + image: pgvector/pgvector:pg16 + ports: + - name: postgres + containerPort: 5432 + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_USER + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_DB + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: POSTGRES_PASSWORD + # Init into a subdir so the PV mount root (which may carry a lost+found) is never PGDATA. + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + readinessProbe: + exec: + command: ["sh", "-c", 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"'] + initialDelaySeconds: 5 + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 12 + livenessProbe: + exec: + command: ["sh", "-c", 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"'] + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 6 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/deploy/k8s/redis.yaml b/deploy/k8s/redis.yaml new file mode 100644 index 0000000..0d829eb --- /dev/null +++ b/deploy/k8s/redis.yaml @@ -0,0 +1,62 @@ +# Redis cache (caching / rate-limit store only — AGENTS.md). In-memory, no persistence +# (--save ""), LRU eviction at 256mb — matches the compose service. The app treats Redis as an +# optional cache, so nothing gates readiness on it. +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: redis +spec: + selector: + app: redis + ports: + - name: redis + port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: redis + image: redis:7-alpine + args: ["redis-server", "--save", "", "--maxmemory", "256mb", "--maxmemory-policy", "allkeys-lru"] + ports: + - name: redis + containerPort: 6379 + readinessProbe: + exec: + command: ["redis-cli", "ping"] + initialDelaySeconds: 3 + periodSeconds: 5 + livenessProbe: + tcpSocket: + port: 6379 + initialDelaySeconds: 10 + periodSeconds: 10 + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "200m" + memory: "320Mi" diff --git a/deploy/k8s/secret.example.yaml b/deploy/k8s/secret.example.yaml new file mode 100644 index 0000000..c88f792 --- /dev/null +++ b/deploy/k8s/secret.example.yaml @@ -0,0 +1,24 @@ +# TEMPLATE ONLY (D4) — this file documents the Secret's shape. NEVER commit the real Secret. +# The real Secret is created out-of-band so no credential ever enters git: +# +# kubectl -n second-brain create secret generic second-brain-secrets \ +# --from-literal=POSTGRES_PASSWORD='' \ +# --from-literal=SECOND_BRAIN_ADMIN_TOKEN='' \ +# --from-literal=SECOND_BRAIN_GEMINI_API_KEY='' \ +# --from-literal=GRAFANA_ADMIN_PASSWORD='' +# +# (CI creates it with throwaway dummy values from workflow env — see .github/workflows/k8s.yml.) +# If you ever render a real Secret to a file, name it deploy/k8s/secret.yaml — it is gitignored. +apiVersion: v1 +kind: Secret +metadata: + name: second-brain-secrets + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain +type: Opaque +stringData: + POSTGRES_PASSWORD: "change-me-postgres-password" + SECOND_BRAIN_ADMIN_TOKEN: "change-me-admin-token" + SECOND_BRAIN_GEMINI_API_KEY: "" # unused while SECOND_BRAIN_LLM_PROVIDER=fake + GRAFANA_ADMIN_PASSWORD: "change-me-grafana-password" diff --git a/deploy/k8s/worker.yaml b/deploy/k8s/worker.yaml new file mode 100644 index 0000000..be34744 --- /dev/null +++ b/deploy/k8s/worker.yaml @@ -0,0 +1,62 @@ +# Durable-job worker (Phase 5, ADR-0013). Same image as the api; runs the resident poll loop +# (`python -m app.jobs.worker --loop`) instead of uvicorn. Drains the jobs queue (daily briefing + +# async research). No ports, no migrations (the migrate Job owns those). Connects via pgbouncer. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: worker + namespace: second-brain + labels: + app.kubernetes.io/part-of: second-brain + app: worker +spec: + replicas: 1 + selector: + matchLabels: + app: worker + template: + metadata: + labels: + app: worker + app.kubernetes.io/part-of: second-brain + spec: + containers: + - name: worker + image: second-brain-api:phase7 + imagePullPolicy: IfNotPresent + command: ["python", "-m", "app.jobs.worker", "--loop"] + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_USER + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + valueFrom: + configMapKeyRef: + name: second-brain-config + key: POSTGRES_DB + - name: SECOND_BRAIN_DATABASE_URL + value: postgresql+psycopg://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@pgbouncer:6432/$(POSTGRES_DB) + - name: SECOND_BRAIN_LLM_PROVIDER + valueFrom: + configMapKeyRef: + name: second-brain-config + key: SECOND_BRAIN_LLM_PROVIDER + - name: SECOND_BRAIN_GEMINI_API_KEY + valueFrom: + secretKeyRef: + name: second-brain-secrets + key: SECOND_BRAIN_GEMINI_API_KEY + resources: + requests: + cpu: "100m" + memory: "512Mi" + limits: + cpu: "500m" + memory: "1Gi" diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 59e8dcf..ef982fa 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -15,7 +15,7 @@ session — the master prompt treats it as the source of truth for "where we are | 4 | MCP server + agentic actions incl. self-research tool | ✅ Complete | | 5 | Daily briefing + scheduled pipelines | ✅ Complete | | 6 | Productionize on VPS + data-ops hardening | ✅ Complete | -| 7 | Kubernetes learning track on local k3s/kind | ⬜ Not started | +| 7 | Kubernetes learning track on local k3s/kind | ✅ Complete | Legend: ⬜ not started · 🟡 in progress · ✅ complete @@ -23,6 +23,48 @@ Legend: ⬜ not started · 🟡 in progress · ✅ complete Add a dated entry per working session. Most recent on top. +### 2026-06-02 — Phase 7 COMPLETE: Kubernetes learning track on local kind (manifests + HPA + ingress + CI/CD), torn down +- **Branch:** `phase-7-impl` (off main, Phase 5 merged via PR #10). Plan in `docs/phase-7-plan.md`; + decisions in **ADR-0014**. NOT pytest-TDD — verification is "apply manifest → assert rollout/health/ + scale", captured as text under **`docs/k8s-evidence/`** (00 overview + 01–09 per layer + 11 teardown). +- **What shipped (`deploy/k8s/`):** real manifests translating all **8** prod-compose services — + Postgres **StatefulSet + PVC** (+ headless Service), a one-shot **migrate Job** (`alembic upgrade + head` direct to the DB, split out of the api's compose command, D3), **pgbouncer** (env-configured + so no `userlist.txt`/password is committed, D12), **redis**, **api** (uvicorn only; CPU requests for + HPA), **worker** (`--loop`), **frontend** (NEXT_PUBLIC baked at build, D11), **ingress-nginx** + host-based routing (`api.second-brain.local` / `second-brain.local`), **metrics-server + HPA** on + api CPU, and **Prometheus + Grafana** (configs reused `--from-file` from Phase 6 — DRY). Plus a + `kustomization.yaml` (one-shot `apply -k`), `secret.example.yaml` (template only, D4), and + `deploy/k8s/README.md` (run/verify/teardown). +- **Verified live on a multi-node kind cluster (1 control-plane + 2 workers, v1.35.0):** all pods + Ready; migrate Job Complete (schema at `0004`); `SELECT 1` through pgbouncer (scram-sha-256); api + `/health` `db:ok`; **worker drained an enqueued briefing job** (queued→done, Briefing row written); + frontend serves HTML with the ingress API host baked into the client bundle; ingress smoke `/health` + 200 + UI 200 (via `Host` header); **HPA scaled api 1→4 under `hey` load** (CPU peaked 400%/50%, + 36,293 reqs all 200, pods spread across both workers) **and back 4→1**; Prometheus scrapes the api + (`up{job=second-brain-api}=1`); Grafana `/api/health` 200. `kubectl apply -k` server-dry-run clean. +- **CI/CD:** new **`.github/workflows/k8s.yml`** (kind-action, pinned kind v0.31.0/node v1.35.0, + ingress-nginx v1.12.3, metrics-server v0.7.2): build+load images → create secret/configmaps → + apply → wait all rollouts → smoke `/health` + UI through ingress → kind-action tears the cluster + down. The eval-gated **`ci.yml` is untouched** (D8). HPA load stays a local evidence step (D13). +- **Decisions (D1–D13, ADR-0014):** multi-node kind; local images via `kind load` + `IfNotPresent` + (no registry, $0); StatefulSet+migrate-Job; ConfigMap/Secret split (secrets uncommitted); host + ingress; HPA-on-CPU; reuse monitoring; new K8s CI; **managed cloud OFF by default (D9)**; teardown + (D10). Added: D11 NEXT_PUBLIC build-time bake, D12 pgbouncer env-config, D13 local HPA evidence. +- **Off-spec fix:** added a root **`.dockerignore`** — without it the repo-root build context shipped + the host `backend/.venv` (1.3G) and `frontend/node_modules` (660M, wrong-OS) INTO the images, + bloating the backend and breaking the frontend. The Phase-6 images were only `docker compose + config`-linted, never built, so this latent bug surfaced on first real build. Detail in + `implementation-notes.md`. +- **Teardown (D10):** `kind delete cluster` — 3 nodes deleted, `No kind clusters found`, no + `second-brain` containers. **Nothing left running ($0).** No managed cloud was ever created. +- **PR:** [#11](https://github.com/tomnguyen103/second-brain/pull/11) — `phase-7-impl` → main; merge + gated on CI (the new `k8s.yml` + the untouched eval-gated `ci.yml`) green **and** the CodeRabbit + (Pro) deep review addressed. +- **Next:** roadmap phases 0–7 all complete. Optional follow-ups: provision the Oracle box + execute + the deploy runbook; CPU-only torch image; the optional managed-cluster (GKE/EKS) capstone (D9, paid + — would need explicit go-ahead and immediate teardown). + ### 2026-06-02 — Phase 5 COMPLETE: daily briefing + scheduled pipelines (durable worker) - **Branch:** `phase-5-impl` (off main, Phase 6 merged via PR #9). Plan in `docs/phase-5-plan.md`; decisions in ADR-0013. Phase 5 was skipped on the 4 → 6 jump; this picks up the deferred items @@ -262,4 +304,7 @@ Add a dated entry per working session. Most recent on top. Vietnam daily-use UI). - ~~Chunking strategy specifics (size/overlap)~~ — RESOLVED in ADR-0003 (~512 tok / ~15% overlap, semantic boundaries). - ~~**Install Docker Desktop** before Phase 1 end-to-end / integration tests~~ — DONE 2026-06-01: installed, Phase 0 migration applied live; Docker DB on host **5433** (native PG holds 5432). -- Whether to do the optional managed-cluster (GKE/EKS) capstone in Phase 7. +- ~~Whether to do the optional managed-cluster (GKE/EKS) capstone in Phase 7~~ — DECIDED 2026-06-02 + in ADR-0014 (D9): **OFF by default** (paid; would blow the $0/learning-track constraint). The local + kind track is the Phase 7 deliverable; a managed-cluster demo stays optional and, if ever run, needs + explicit go-ahead and immediate teardown. diff --git a/docs/adr/0014-kubernetes-learning-track.md b/docs/adr/0014-kubernetes-learning-track.md new file mode 100644 index 0000000..52c71a4 --- /dev/null +++ b/docs/adr/0014-kubernetes-learning-track.md @@ -0,0 +1,76 @@ +# ADR-0014: Kubernetes learning track on local kind (manifests + HPA + ingress + CI/CD), then torn down + +- **Status:** Accepted +- **Date:** 2026-06-02 +- **Phase:** 7 +- **Supersedes / relates:** ADR-0011 (VPS = production runtime), ADR-0012 (productionization). This + ADR does **not** change the production runtime — it adds a *learning track*. + +## Context + +AGENTS.md fixes the production runtime as **one small VPS running Docker Compose** (ADR-0011/0012): +a single-user app gets no benefit from Kubernetes' multi-node scheduling/autoscaling/self-healing, +and managed K8s would blow the ~$5/mo budget to $70+/mo. But "can operate the app on real +Kubernetes" is a signal worth demonstrating. The roadmap therefore carved out Phase 7 as a +**learning track**: prove the stack runs on real K8s with proper manifests, then tear it down so it +costs nothing and is never the prod runtime. The input was the 8-service prod compose stack +(`deploy/docker-compose.prod.yml`): `db`, `pgbouncer`, `redis`, `api`, `worker`, `frontend`, +`prometheus`, `grafana`. + +## Decision + +Translate the stack to real Kubernetes manifests (`deploy/k8s/`) and prove it on a free, local, +**multi-node `kind`** cluster, capturing evidence per layer (`docs/k8s-evidence/`), then **delete the +cluster** (D10). Key decisions (full list D1–D13 in `docs/phase-7-plan.md`): + +- **D1** Cluster = multi-node `kind` (1 control-plane + 2 workers) so HPA pod-spread and + ingress-on-a-labelled-node are real, not single-node toys. +- **D2** Images built locally and `kind load`ed (no registry, $0); fixed `:phase7` tag + + `imagePullPolicy: IfNotPresent` so K8s never attempts a registry pull. +- **D3** Postgres = StatefulSet + PVC; `alembic upgrade head` runs as a one-shot **Job** against the + DB directly (not via pgbouncer) — split out of the api's compose `command`. +- **D4** Config split: ConfigMap (non-secret) + Secret (DB password, admin token, Gemini key, + Grafana password). **Secrets are never committed** — only `secret.example.yaml`; the real Secret + is created out-of-band. The password stays out of DSNs by `$(VAR)` env assembly per pod. +- **D5** Ingress = ingress-nginx, host-based (`api.second-brain.local`, `second-brain.local`). +- **D6** Autoscaling = metrics-server + HPA on api CPU; demonstrated under `hey` load. +- **D7** Observability = Prometheus + Grafana reused verbatim from Phase 6 configs (no RAM trim needed). +- **D8** New CI workflow `k8s.yml` (kind-action) stands the stack up and tears it down; the + eval-gated `ci.yml` is untouched. +- **D9** Managed cloud (GKE/EKS) = OFF by default; no paid resource without explicit approval. +- **D10** Teardown after evidence: `kind delete cluster`. +- **D11 (added)** `NEXT_PUBLIC_API_BASE_URL` is build-time baked (Next inlines `NEXT_PUBLIC_*` at + `next build`); `Dockerfile.frontend` gains an additive, default-preserving build `ARG` so the K8s + image bakes the ingress API host. +- **D12 (added)** pgbouncer configured by env (edoburu auto-generates `userlist.txt`) so the DB + password comes from the Secret, never a committed `userlist.txt`; SESSION pool mode kept. +- **D13 (added)** HPA load-scaling is proven **locally** (evidence 08); CI does + build→load→apply→rollout→smoke→teardown only (deterministic, no flaky timing). + +## Consequences + +**Good** +- A real, reproducible K8s proof: StatefulSet+PVC, migrate Job, Deployments, ingress, HPA (scaled + api 1→4 under load and back), Prometheus scraping the api, Grafana healthy — all captured. +- Manifests + a kind CI workflow are committed and recreate the stack on demand; nothing runs idle. +- Surfaced + fixed a latent Phase-6 image bug (no `.dockerignore` → host `.venv`/`node_modules` + shipped into the images); the prod images were only `docker compose config`-linted before, never built. +- Demonstrates engineering judgment: knowing when **not** to run K8s in production is itself a signal. + +**Bad / trade-offs** +- The manifests are a learning artifact, not the prod runtime — they drift from compose unless kept + in sync (mitigated: monitoring configs are reused `--from-file`, not duplicated). +- The backend image carries CUDA torch wheels (large); fine on kind, but a CPU-only torch build + would slim it — deferred (it's the existing Phase-1 requirements, out of Phase 7 scope). +- HPA scaling isn't gated in CI (D13) — demonstrated locally instead. + +## Alternatives rejected + +- **Run K8s in production (managed GKE/EKS).** Rejected: cost ($70+/mo vs ~$5) and complexity for a + single-user app. (D9 keeps a managed-cluster capstone optional and off by default.) +- **k3s instead of kind.** Either works for the track; kind is the lighter, throwaway, + CI-native choice (kind-action) and needs only the already-installed Docker Desktop. +- **Commit a rendered Secret / a pgbouncer `userlist.txt`.** Rejected: would leak credentials into + git (D4/D12). +- **Single-node kind.** Rejected: HPA spread and ingress-ready node scheduling are more honest on + multi-node (D1). diff --git a/docs/adr/README.md b/docs/adr/README.md index b847b35..2e03799 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -19,3 +19,4 @@ older ones explicitly; we don't edit history. | [0011](0011-vps-provider.md) | VPS provider: Oracle Always Free (Singapore) primary, Contabo SG paid fallback | Accepted | | [0012](0012-productionization-and-data-governance.md) | Productionization + data governance: RLS, audit, retention, GDPR erasure, metrics, eval-gated CI, PgBouncer | Accepted | | [0013](0013-briefing-scheduling-and-worker.md) | Daily briefing: durable poll-based worker, OS-cron scheduling, store-and-display, briefings table | Accepted | +| [0014](0014-kubernetes-learning-track.md) | Kubernetes learning track on local kind (manifests + HPA + ingress + CI/CD), then torn down | Accepted | diff --git a/docs/implementation-notes.md b/docs/implementation-notes.md index 6ff8187..57deb80 100644 --- a/docs/implementation-notes.md +++ b/docs/implementation-notes.md @@ -9,6 +9,75 @@ what I gave up**. Keep it honest — the surprises are the valuable part. --- +## Phase 7 — Kubernetes learning track on local kind (2026-06-02) + +### Root `.dockerignore` added — the prod images were never actually built before +- **What:** added `/.dockerignore` excluding `**/.venv/`, `**/node_modules/`, `.git/`, `**/.next/`, + `**/mlruns/`, `**/.env`, agent dirs. +- **Why:** the `deploy/Dockerfile.*` build context is the **repo root**, and both Dockerfiles do + `COPY backend/ ./` / `COPY frontend/ ./`. With no `.dockerignore`, Docker shipped the host + `backend/.venv` (1.3G, Windows) and `frontend/node_modules` (660M, wrong-OS native binaries) as + context AND copied them into the images — bloating the backend image and **breaking** the frontend + image (platform-mismatched modules). Phase 6 only ran `docker compose config` (lint), never a real + build, so this latent bug first surfaced when Phase 7 actually built the images. +- **Trade-off:** none — strictly correct. *Affects:* `/.dockerignore`, both images. + +### `NEXT_PUBLIC_API_BASE_URL` is build-time baked → additive `ARG` on Dockerfile.frontend (D11) +- **What / why:** Next.js inlines `NEXT_PUBLIC_*` at `next build`; a runtime ConfigMap value is + ignored by the browser bundle (`frontend/lib/api/client.ts` reads `process.env.NEXT_PUBLIC_API_BASE_URL`). + For K8s the browser must call the API's **ingress** host, so `Dockerfile.frontend` gained + `ARG NEXT_PUBLIC_API_BASE_URL=http://localhost:8000` + `ENV` before `npm run build`; the K8s image + is built with `--build-arg ...=http://api.second-brain.local`. Verified the host is in BOTH the + client and server `.next` bundles. +- **Trade-off:** one Phase-6 file touched, but **additively** (default preserves compose behaviour). + *Affects:* `deploy/Dockerfile.frontend`, `deploy/k8s/frontend.yaml`. + +### pgbouncer configured by env, not a committed `userlist.txt` (D12) +- **What / why:** the compose stack mounts `pgbouncer/userlist.txt` (which embeds the DB credential). + Committing that to a manifest/ConfigMap would leak a secret, so the K8s pgbouncer uses + `edoburu/pgbouncer`'s env config (`DB_HOST/DB_USER/DB_PASSWORD` from the Secret; it auto-generates + the userlist). `AUTH_TYPE=scram-sha-256` (PG16 default) + `POOL_MODE=session` (psycopg3 prepared + statements, ADR-0012). Verified `SELECT 1` through `pgbouncer:6432`. +- **Trade-off:** diverges from the file-mounted compose config; keeps the credential in the Secret only. + +### Migrations are a Job; the password stays out of DSNs via `$(VAR)` assembly (D3/D4) +- **What:** the api compose `command` prefixes `alembic upgrade head`; in K8s that's a one-shot + **Job** (`migrate-job.yaml`) talking to `db:5432` directly, and the api/worker run only their + process. Each pod assembles `SECOND_BRAIN_DATABASE_URL` from `POSTGRES_USER/PASSWORD/DB` (Secret + + ConfigMap) via Kubernetes dependent-env `$(VAR)` substitution, so the password never appears in a + ConfigMap or a committed DSN. `extra="ignore"` in `Settings` means the helper `POSTGRES_*` env vars + are harmless to the app. +- **Trade-off:** `apply -k` doesn't order resources, so the Job can start before Postgres is Ready — + covered by `restartPolicy: OnFailure` + `backoffLimit` (and the layered path waits for db first). + +### Worker eager-loads the embedder; api lazy-loads it +- **What / why:** `worker.main()` calls `get_embedder()` at startup (loads MiniLM), so the worker pod + carries the model resident; the api loads it lazily on first ingest/chat (`/health` reported + `embedder:"unloaded"`, RSS ~88Mi). Drove the HPA with `/health` (no model load) so scaling reflects + request-handling CPU, not a one-time model load. *Affects:* HPA load target choice (D6). + +### HPA load-scaling proven locally, not in CI (D13) +- **What / why:** a load-driven autoscale is timing-sensitive and the torch image is slow to build + + RAM-heavy per replica, so asserting "pods went 1→N" in CI would be flaky. CI proves the manifests + stand up and serve (`k8s.yml`); the scaling proof is the local evidence (`docs/k8s-evidence/08`, + api 1→4→1 under `hey`). *Trade-off:* CI doesn't gate scaling — acceptable for a learning track. + +### kind cluster name via `--name` flag only; monitoring ConfigMaps via `--from-file` +- **What / why (name):** `helm/kind-action` passes `--name`, and kind rejects a name set in **both** + the config and the flag. Removed `name:` from `kind-cluster.yaml` so the name is supplied only by + `--name second-brain` (local command + CI `cluster_name`), keeping local and CI aligned. +- **What / why (configmaps):** kustomize's `configMapGenerator` refuses file sources above the + kustomization root (`../prometheus/...`), and `kubectl apply -k` has no `--load-restrictor`. So the + Prometheus/Grafana configs stay an explicit `kubectl create configmap --from-file ... | apply -f -` + step (reusing the Phase 6 files — DRY, no duplicated copy), documented in the kustomization header, + README, and CI. *Affects:* `deploy/k8s/kustomization.yaml`, `deploy/k8s/README.md`, `k8s.yml`. + +### Pinned versions for reproducibility +- kind **v0.31.0** → node **kindest/node:v1.35.0** (kubectl v1.34 client ↔ v1.35 server is within the + ±1 skew); ingress-nginx **controller-v1.12.3**; metrics-server **v0.7.2** (+ `--kubelet-insecure-tls`, + required on kind). The manifests use only stable APIs (`apps/v1`, `batch/v1`, `autoscaling/v2`, + `networking.k8s.io/v1`), so they're robust across these versions. + ## Phase 5 — daily briefing + scheduled pipelines (2026-06-02) ### Worker transaction model: one commit per attempt, queue primitives only flush diff --git a/docs/k8s-evidence/00-stack-overview.txt b/docs/k8s-evidence/00-stack-overview.txt new file mode 100644 index 0000000..95c8750 --- /dev/null +++ b/docs/k8s-evidence/00-stack-overview.txt @@ -0,0 +1,43 @@ +Phase 7 evidence — OVERVIEW: full Second Brain stack running on local kind +========================================================================== +All 8 Compose services translated to K8s and running, plus the migrate Job complete, ingress +serving, and HPA active. Cluster is ephemeral (torn down after capture, D10) — this is the proof. + +$ kubectl -n second-brain get deploy,statefulset,job,svc,ingress,hpa +NAME READY UP-TO-DATE AVAILABLE AGE +deployment.apps/api 1/1 1 1 14m +deployment.apps/frontend 1/1 1 1 12m +deployment.apps/grafana 1/1 1 1 3m5s +deployment.apps/pgbouncer 1/1 1 1 15m +deployment.apps/prometheus 1/1 1 1 3m6s +deployment.apps/redis 1/1 1 1 15m +deployment.apps/worker 1/1 1 1 14m + +NAME READY AGE +statefulset.apps/db 1/1 27m + +NAME STATUS COMPLETIONS DURATION AGE +job.batch/migrate Complete 1/1 5s 16m + +NAME TYPE CLUSTER-IP PORT(S) AGE +service/api ClusterIP 10.96.239.234 8000/TCP 14m +service/db ClusterIP None 5432/TCP 27m (headless — StatefulSet) +service/frontend ClusterIP 10.96.205.62 3000/TCP 12m +service/grafana ClusterIP 10.96.98.140 3000/TCP 3m5s +service/pgbouncer ClusterIP 10.96.154.51 6432/TCP 15m +service/prometheus ClusterIP 10.96.47.198 9090/TCP 3m6s +service/redis ClusterIP 10.96.183.8 6379/TCP 15m + +NAME CLASS HOSTS ADDRESS PORTS +ingress.networking.k8s.io/second-brain nginx api.second-brain.local,second-brain.local localhost 80 + +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS +horizontalpodautoscaler.autoscaling/api Deployment/api cpu: 0%/50% 1 4 1 + +# Pods spread across both worker nodes; app pods run the local kind-loaded images (no registry): + api / migrate / worker -> second-brain-api:phase7 (built from deploy/Dockerfile.backend) + frontend -> second-brain-web:phase7 (built from deploy/Dockerfile.frontend, D11) + db -> pgvector/pgvector:pg16 | pgbouncer -> edoburu/pgbouncer | redis -> redis:7-alpine + prometheus -> prom/prometheus | grafana -> grafana/grafana + +Per-layer gate evidence: see 01..09 in this directory. Teardown proof: see 11-teardown.txt. diff --git a/docs/k8s-evidence/01-cluster-and-namespace.txt b/docs/k8s-evidence/01-cluster-and-namespace.txt new file mode 100644 index 0000000..7d94ac1 --- /dev/null +++ b/docs/k8s-evidence/01-cluster-and-namespace.txt @@ -0,0 +1,29 @@ +Phase 7 evidence — Task 1: multi-node kind cluster + namespace +================================================================ +Captured on the local learning-track cluster (kind v0.31.0, node image kindest/node:v1.35.0). +The cluster is ephemeral (torn down at the end, D10); these captures are the durable proof. + +$ kind create cluster --config deploy/k8s/kind-cluster.yaml + • Ensuring node image (kindest/node:v1.35.0) ... ✓ + • Preparing nodes 📦 📦 📦 ... ✓ + • Starting control-plane ... ✓ + • Installing CNI ... ✓ + • Installing StorageClass ... ✓ + • Joining worker nodes ... ✓ +Set kubectl context to "kind-second-brain" + +$ kubectl wait --for=condition=Ready nodes --all --timeout=120s +node/second-brain-control-plane condition met +node/second-brain-worker condition met +node/second-brain-worker2 condition met + +$ kubectl get nodes -o wide +NAME STATUS ROLES AGE VERSION INTERNAL-IP OS-IMAGE CONTAINER-RUNTIME +second-brain-control-plane Ready control-plane 31s v1.35.0 172.19.0.3 Debian GNU/Linux 12 (bookworm) containerd://2.2.0 +second-brain-worker Ready 18s v1.35.0 172.19.0.4 Debian GNU/Linux 12 (bookworm) containerd://2.2.0 +second-brain-worker2 Ready 18s v1.35.0 172.19.0.2 Debian GNU/Linux 12 (bookworm) containerd://2.2.0 + +$ kubectl apply -f deploy/k8s/namespace.yaml +namespace/second-brain created + +GATE PASSED: control-plane + 2 workers Ready; namespace second-brain Active. diff --git a/docs/k8s-evidence/02-config-and-secrets.txt b/docs/k8s-evidence/02-config-and-secrets.txt new file mode 100644 index 0000000..174b3d5 --- /dev/null +++ b/docs/k8s-evidence/02-config-and-secrets.txt @@ -0,0 +1,26 @@ +Phase 7 evidence — Task 2: ConfigMap + Secret (secrets NOT committed, D4) +========================================================================= +Non-secret config is a ConfigMap; the 4 secret values are a Secret created imperatively +(no credential file on disk, nothing secret in git). Only secret.example.yaml is committed. + +$ kubectl apply -f deploy/k8s/configmap.yaml +configmap/second-brain-config created + +$ kubectl -n second-brain create secret generic second-brain-secrets \ + --from-literal=POSTGRES_PASSWORD='***' \ + --from-literal=SECOND_BRAIN_ADMIN_TOKEN='***' \ + --from-literal=SECOND_BRAIN_GEMINI_API_KEY='' \ + --from-literal=GRAFANA_ADMIN_PASSWORD='***' +secret/second-brain-secrets created + +$ kubectl -n second-brain get configmap second-brain-config -o jsonpath='{.data}' +{"POSTGRES_DB":"second_brain","POSTGRES_USER":"second_brain","SECOND_BRAIN_LLM_PROVIDER":"fake"} + +$ kubectl -n second-brain get secret second-brain-secrets -o jsonpath='{.data}' (keys only — values redacted) +GRAFANA_ADMIN_PASSWORD +POSTGRES_PASSWORD +SECOND_BRAIN_ADMIN_TOKEN +SECOND_BRAIN_GEMINI_API_KEY + +GATE PASSED: ConfigMap (3 non-secret keys) + Secret (4 keys) present in the namespace. +git status confirms no real Secret is staged (only configmap.yaml + secret.example.yaml committed). diff --git a/docs/k8s-evidence/03-postgres-and-migrate.txt b/docs/k8s-evidence/03-postgres-and-migrate.txt new file mode 100644 index 0000000..6b62ac0 --- /dev/null +++ b/docs/k8s-evidence/03-postgres-and-migrate.txt @@ -0,0 +1,42 @@ +Phase 7 evidence — Task 3: Postgres StatefulSet + PVC + migrate Job (D3) +======================================================================== +Images built locally and loaded into all kind nodes (no registry, D2): + $ docker build -f deploy/Dockerfile.backend -t second-brain-api:phase7 . # exit 0 + $ docker build -f deploy/Dockerfile.frontend -t second-brain-web:phase7 \ + --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local . # exit 0 (D11) + $ kind load docker-image second-brain-api:phase7 --name second-brain # loaded x3 nodes + $ kind load docker-image second-brain-web:phase7 --name second-brain # loaded x3 nodes +(.dockerignore was added so the host .venv/node_modules are NOT shipped into the images.) + +$ kubectl apply -f deploy/k8s/postgres-statefulset.yaml +service/db created +statefulset.apps/db created + +$ kubectl -n second-brain rollout status statefulset/db --timeout=240s +partitioned roll out complete: 1 new pods have been updated... + +$ kubectl -n second-brain get pods -l app=db -o wide +NAME READY STATUS RESTARTS AGE IP NODE +db-0 1/1 Running 0 22s 10.244.2.3 second-brain-worker + +$ kubectl -n second-brain get pvc +NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS +data-db-0 Bound pvc-f2de6cac-d66a-4255-acc6-ef38d7da6b02 1Gi RWO standard + +$ kubectl apply -f deploy/k8s/migrate-job.yaml +job.batch/migrate created + +$ kubectl -n second-brain wait --for=condition=complete job/migrate --timeout=180s +job.batch/migrate condition met + +$ kubectl -n second-brain get job migrate +NAME STATUS COMPLETIONS DURATION AGE +migrate Complete 1/1 5s 5s + +$ kubectl -n second-brain logs job/migrate +INFO [alembic.runtime.migration] Running upgrade -> 0001_baseline, baseline schema ... + pgvector/full-text indexes +INFO [alembic.runtime.migration] Running upgrade 0001_baseline -> 0002_tasks +INFO [alembic.runtime.migration] Running upgrade 0002_tasks -> 0003_rls_audit +INFO [alembic.runtime.migration] Running upgrade 0003_rls_audit -> 0004_briefings + +GATE PASSED: Postgres pod Ready + PVC Bound; migrate Job Complete; schema at head (0004_briefings). diff --git a/docs/k8s-evidence/04-pgbouncer-and-redis.txt b/docs/k8s-evidence/04-pgbouncer-and-redis.txt new file mode 100644 index 0000000..8fb4d23 --- /dev/null +++ b/docs/k8s-evidence/04-pgbouncer-and-redis.txt @@ -0,0 +1,25 @@ +Phase 7 evidence — Task 4: pgbouncer (env-config, D12) + redis +============================================================== +pgbouncer is configured entirely by env (no committed userlist.txt); the DB password comes from +the Secret. scram-sha-256 client auth + session pooling verified by a real query THROUGH pgbouncer. + +$ kubectl apply -f deploy/k8s/pgbouncer.yaml +$ kubectl apply -f deploy/k8s/redis.yaml +$ kubectl -n second-brain rollout status deploy/pgbouncer --timeout=120s +deployment "pgbouncer" successfully rolled out +$ kubectl -n second-brain rollout status deploy/redis --timeout=120s +deployment "redis" successfully rolled out + +$ kubectl -n second-brain get pods -l 'app in (pgbouncer,redis)' +NAME READY STATUS RESTARTS AGE +pgbouncer-5d5cbd46dc-ls75n 1/1 Running 0 11s +redis-7f697f78cc-h5hq5 1/1 Running 0 11s + +# Real query through the pooler (client -> pgbouncer:6432 -> db:5432), proving auth + pooling: +$ kubectl -n second-brain exec db-0 -- psql "postgresql://second_brain:***@pgbouncer:6432/second_brain" -tAc "SELECT 1 AS through_pgbouncer" +1 + +$ kubectl -n second-brain exec deploy/redis -- redis-cli ping +PONG + +GATE PASSED: both Running; SELECT 1 succeeds through pgbouncer (scram-sha-256, session mode); redis PONG. diff --git a/docs/k8s-evidence/05-api-and-worker.txt b/docs/k8s-evidence/05-api-and-worker.txt new file mode 100644 index 0000000..9d40218 --- /dev/null +++ b/docs/k8s-evidence/05-api-and-worker.txt @@ -0,0 +1,38 @@ +Phase 7 evidence — Task 5: api + worker Deployments +==================================================== +api runs only uvicorn (migrations are the Job's, D3) and reaches Postgres through pgbouncer. +worker runs the resident --loop and actually drains an enqueued job (proved via the DB, since +run_loop is silent on success). NOTE: the worker EAGER-loads the embedder at startup (main() +calls get_embedder()); the api LAZY-loads it (embedder:"unloaded" below). + +$ kubectl apply -f deploy/k8s/api.yaml ; kubectl apply -f deploy/k8s/worker.yaml +$ kubectl -n second-brain rollout status deploy/api --timeout=240s +deployment "api" successfully rolled out +$ kubectl -n second-brain rollout status deploy/worker --timeout=180s +deployment "worker" successfully rolled out + +$ kubectl -n second-brain get pods -l 'app in (api,worker)' -o wide +NAME READY STATUS RESTARTS AGE NODE +api-c6bc96c9d-z4v4l 1/1 Running 0 13s second-brain-worker +worker-66f499988d-hl27m 1/1 Running 0 12s second-brain-worker2 (ready=true, restarts=0) + +# api /health from inside the pod — db:ok proves it reaches Postgres via pgbouncer: +$ kubectl -n second-brain exec deploy/api -- python -c "import urllib.request; print(urllib.request.urlopen('http://localhost:8000/health').read().decode())" +{"status":"ok","db":"ok","embedder":"unloaded"} + +# Prove the worker genuinely processes jobs on K8s: enqueue a briefing, then check the DB. +$ kubectl -n second-brain exec deploy/api -- python -m app.jobs.enqueue briefing +enqueued job 1 (briefing) +# (worker polls every 5s; waited ~12s) +$ psql ... -c "SELECT id, type, status, attempts FROM jobs ORDER BY id" + id | type | status | attempts +----+----------+--------+---------- + 1 | briefing | done | 0 +$ psql ... -c "SELECT id, document_count, length(body_markdown) AS body_len, period_end FROM briefings" + id | document_count | body_len | period_end +----+----------------+----------+------------------------------- + 1 | 0 | 179 | 2026-06-02 16:26:24.382139+00 +# document_count=0 (empty corpus -> "nothing new" briefing, no LLM call) — expected Phase 5 behaviour. + +GATE PASSED: api Ready + /health db:ok; worker Ready (0 restarts) and drained a real job +(queued -> done, Briefing row persisted) end-to-end on the cluster. diff --git a/docs/k8s-evidence/06-frontend.txt b/docs/k8s-evidence/06-frontend.txt new file mode 100644 index 0000000..aeb24a1 --- /dev/null +++ b/docs/k8s-evidence/06-frontend.txt @@ -0,0 +1,25 @@ +Phase 7 evidence — Task 6: frontend Deployment (NEXT_PUBLIC baked, D11) +======================================================================= +The frontend image was built with --build-arg NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local +so Next inlines the ingress API host at build time (a runtime ConfigMap value would be ignored). + +$ kubectl apply -f deploy/k8s/frontend.yaml +service/frontend created +deployment.apps/frontend created +$ kubectl -n second-brain rollout status deploy/frontend --timeout=180s +deployment "frontend" successfully rolled out + +$ kubectl -n second-brain get pods -l app=frontend +NAME READY STATUS RESTARTS AGE +frontend-587754f8c7-xsc2g 1/1 Running 0 13s + +# Serves the Next.js app shell: +$ kubectl -n second-brain exec deploy/frontend -- wget -qO- http://localhost:3000/ + ... + +# D11 bake verified — the ingress API host is in BOTH client and server bundles: +$ kubectl -n second-brain exec deploy/frontend -- sh -c "grep -rl 'api.second-brain.local' .next | head -3" +.next/server/chunks/ssr/_0q07iul._.js +.next/static/chunks/1vq19va-s4k-8.js <-- client bundle: the browser will call the API via ingress + +GATE PASSED: frontend Ready, serves HTML, and the API base URL is baked into the client bundle (D11). diff --git a/docs/k8s-evidence/07-ingress.txt b/docs/k8s-evidence/07-ingress.txt new file mode 100644 index 0000000..7f25885 --- /dev/null +++ b/docs/k8s-evidence/07-ingress.txt @@ -0,0 +1,31 @@ +Phase 7 evidence — Task 7: ingress-nginx host-based routing (D5) +================================================================ +ingress-nginx (kind provider, PINNED controller-v1.12.3) routes by host: + api.second-brain.local -> api:8000 , second-brain.local -> frontend:3000. +The controller binds the control-plane node's host ports 80/443 (kind extraPortMappings), so the +hosts are reachable at http://localhost. Smoke uses a Host header (no /etc/hosts edit needed). + +$ kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.3/deploy/static/provider/kind/deploy.yaml +$ kubectl wait -n ingress-nginx --for=condition=ready pod -l app.kubernetes.io/component=controller --timeout=180s +pod/ingress-nginx-controller-7dd5849869-ppdxf condition met + +$ kubectl apply -f deploy/k8s/ingress.yaml +ingress.networking.k8s.io/second-brain created +$ kubectl -n second-brain get ingress +NAME CLASS HOSTS PORTS AGE +second-brain nginx api.second-brain.local,second-brain.local 80 0s + +# --- API through ingress --- +$ curl -s -H "Host: api.second-brain.local" http://localhost/health (HTTP 200) +{"status":"ok","db":"ok","embedder":"unloaded"} + +# --- UI through ingress --- ('/' redirects to /chat by app design; /chat renders) --- +$ curl -s -i -H "Host: second-brain.local" http://localhost/ +HTTP/1.1 307 Temporary Redirect +location: /chat +$ curl -s -H "Host: second-brain.local" http://localhost/chat (HTTP 200) + ... +$ curl -s -L -H "Host: second-brain.local" http://localhost/ (HTTP 200 after following) + +GATE PASSED: ingress serves the api (/health 200, db:ok) AND the UI (/, 307->/chat, 200 HTML). +For a browser, add: 127.0.0.1 second-brain.local api.second-brain.local diff --git a/docs/k8s-evidence/08-hpa-autoscaling.txt b/docs/k8s-evidence/08-hpa-autoscaling.txt new file mode 100644 index 0000000..c3bedf7 --- /dev/null +++ b/docs/k8s-evidence/08-hpa-autoscaling.txt @@ -0,0 +1,42 @@ +Phase 7 evidence — Task 8: metrics-server + HPA autoscaling under load (D6) +=========================================================================== +metrics-server v0.7.2 installed with --kubelet-insecure-tls (required on kind). HPA on api CPU +(target 50% of the 250m request, min 1 / max 4). Load driven by an in-cluster `hey` pod at +/health. Autoscaling demonstrated BOTH directions (1 -> 4 under load, 4 -> 1 after). + +$ kubectl top nodes # metrics-server working +NAME CPU(cores) CPU(%) MEMORY(bytes) MEMORY(%) +second-brain-control-plane 83m 0% 1346Mi 4% +second-brain-worker 42m 0% 579Mi 1% +second-brain-worker2 21m 0% 485Mi 1% + +$ kubectl -n second-brain get hpa api # initial +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS +api Deployment/api cpu: 0%/50% 1 4 1 + +# Load: hey -z 90s -c 80 http://api:8000/health (in-cluster pod). Polling HPA + api pod count: +t= 24s | api pods=3 | cpu: 141%/50% | replicas 1->scaling +t= 40s | api pods=4 | cpu: 400%/50% | replicas 3 +t= 56s | api pods=4 | cpu: 133%/50% | replicas 4 (capped at maxReplicas) +t= 72s | api pods=4 | cpu: 100%/50% | replicas 4 +t= 88s | api pods=4 | cpu: 100%/50% | replicas 4 + +$ kubectl -n second-brain get pods -l app=api -o wide # 4 pods, spread across BOTH workers +api-c6bc96c9d-fg5dp 1/1 Running second-brain-worker +api-c6bc96c9d-z4v4l 1/1 Running second-brain-worker +api-c6bc96c9d-8hz47 1/1 Running second-brain-worker2 +api-c6bc96c9d-knfdt 1/1 Running second-brain-worker2 + +$ kubectl -n second-brain logs load # hey summary + Total: 90.1578 secs + Requests/sec: 402.5498 + Status code distribution: [200] 36293 responses + +# After deleting the load pod, the HPA scales back down (60s stabilization, then 1 pod/30s): +$ kubectl -n second-brain get hpa api # ~150s after load removed +NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS +api Deployment/api cpu: 0%/50% 1 4 1 +api pods now: 1 + +GATE PASSED: HPA scaled api 1 -> 4 under load (CPU peaked 400%/50%, capped at max=4, pods spread +across both worker nodes), then 4 -> 1 after load removed. 36,293 requests served (all 200). diff --git a/docs/k8s-evidence/09-monitoring.txt b/docs/k8s-evidence/09-monitoring.txt new file mode 100644 index 0000000..f588225 --- /dev/null +++ b/docs/k8s-evidence/09-monitoring.txt @@ -0,0 +1,31 @@ +Phase 7 evidence — Task 9: Prometheus + Grafana (D7) +===================================================== +Reuses the Phase 6 configs verbatim (deploy/prometheus/*, deploy/grafana/*) via ConfigMaps +created --from-file (DRY). No local RAM trim needed — node usage stayed low (~4%). + +$ kubectl -n second-brain create configmap prometheus-config \ + --from-file=prometheus.yml=deploy/prometheus/prometheus.yml \ + --from-file=alerts.yml=deploy/prometheus/alerts.yml --dry-run=client -o yaml | kubectl apply -f - +configmap/prometheus-config created +$ # + grafana-datasources, grafana-dashboard-provider, grafana-dashboard-json (from deploy/grafana/*) +$ kubectl apply -f deploy/k8s/monitoring/prometheus.yaml +$ kubectl apply -f deploy/k8s/monitoring/grafana.yaml +$ kubectl -n second-brain rollout status deploy/prometheus --timeout=180s # success +$ kubectl -n second-brain rollout status deploy/grafana --timeout=180s # success + +$ kubectl -n second-brain get pods -l 'app in (prometheus,grafana)' +NAME READY STATUS RESTARTS AGE +grafana-8699c54ddc-gkpnp 1/1 Running 0 59s +prometheus-799d757588-lfd2k 1/1 Running 0 60s + +# Prometheus is scraping the api /metrics — BOTH targets up (value "1"): +$ kubectl -n second-brain exec pq -- curl -s 'http://prometheus:9090/api/v1/query?query=up' +... {"metric":{"instance":"localhost:9090","job":"prometheus"},"value":[..,"1"]}, + {"metric":{"instance":"api:8000","job":"second-brain-api"},"value":[..,"1"]} ... + +# Grafana healthy (auto-provisioned Prometheus datasource + the second-brain dashboard): +$ kubectl -n second-brain exec pq -- curl -s http://grafana:3000/api/health +HTTP 200 +{ "database": "ok", "version": "13.0.2", "commit": "3fcdbc5a" } + +GATE PASSED: prometheus + grafana Ready; api target up=1 in Prometheus; Grafana health 200. diff --git a/docs/k8s-evidence/11-teardown.txt b/docs/k8s-evidence/11-teardown.txt new file mode 100644 index 0000000..6d9fdfb --- /dev/null +++ b/docs/k8s-evidence/11-teardown.txt @@ -0,0 +1,22 @@ +Phase 7 evidence — Task 11: teardown (D10 — leave nothing running, $0) +====================================================================== +After all evidence was captured, the ephemeral learning-track cluster was deleted. Kubernetes was +NEVER the production runtime (that stays the single-VPS Docker Compose stack). No managed cloud was +ever created (D9 — OFF by default). Local kind costs $0 whether up or down; this confirms hygiene. + +$ kind get clusters +second-brain + +$ kind delete cluster --name second-brain +Deleting cluster "second-brain" ... +Deleted nodes: ["second-brain-worker" "second-brain-worker2" "second-brain-control-plane"] + +$ kind get clusters +No kind clusters found. + +$ docker ps -a --filter "name=second-brain" --format "{{.Names}}" +(none) + +GATE PASSED: cluster torn down; no kind clusters; no second-brain containers. Nothing left running. +The committed manifests + CI workflow (k8s.yml) recreate the whole stack on demand; CI also tears +its own cluster down in the kind-action post step. diff --git a/docs/phase-7-plan.md b/docs/phase-7-plan.md new file mode 100644 index 0000000..b05826c --- /dev/null +++ b/docs/phase-7-plan.md @@ -0,0 +1,227 @@ +# Phase 7 — Kubernetes learning track (local kind) Implementation Plan + +> **Not pytest-TDD.** Verification here is **"apply manifest → assert rollout / health / scale"**, +> not red→green. Each task ends with an explicit *verify gate* (a command + the expected +> observation). Build incrementally, apply+verify each layer before the next, commit per green +> layer. Capture evidence as text (`kubectl` output) under `docs/k8s-evidence/`. + +**Goal (per AGENTS.md):** prove the Second Brain stack runs on **real Kubernetes** — proper +manifests, a Postgres StatefulSet, a migrate Job, Deployments for api/worker/frontend, host-based +**ingress**, **HPA** autoscaling under load, reused **Prometheus + Grafana**, and a **CI/CD** +workflow that stands the whole thing up on `kind` and tears it down — then **`kind delete cluster`** +so **nothing is left running ($0)**. Kubernetes is a **LEARNING TRACK only**, *not* the production +runtime (that stays the single-VPS Docker Compose stack, ADR-0011/0012). Managed cloud (GKE/EKS) is +**off by default** (D9) — no paid resource without an explicit OK. + +**Scope reality:** this mirrors how Phases 1–6 deferred their out-of-scope tails. The deliverable is +**manifests + CI + docs + captured evidence**, proven on a *throwaway* local cluster. The cluster +itself is ephemeral; the committed artifacts are the durable output. We translate the 8 prod-compose +services (`deploy/docker-compose.prod.yml`): `db`, `pgbouncer`, `redis`, `api`, `worker`, `frontend`, +`prometheus`, `grafana`. + +**Architecture (Compose → K8s mapping):** + +| Compose service | K8s object(s) | Notes | +|---|---|---| +| `db` (pgvector:pg16) | **StatefulSet** + headless Service + **PVC** | `pg_isready` readiness; `db` Service DNS = `db.second-brain.svc` | +| (migrations) | **Job** `migrate` (`alembic upgrade head`) | Was the api compose `command` prefix; now its own Job → DB **directly** (not via pgbouncer), per D3 | +| `pgbouncer` | **Deployment** + Service `:6432` | `edoburu/pgbouncer` configured by **env** (DB_HOST/USER/PASSWORD) — auto-generates userlist, so no password file is committed (divergence from compose's mounted `userlist.txt`; see D12) | +| `redis` | **Deployment** + Service `:6379` | In-memory, `allkeys-lru`; app treats it as optional cache (no readiness gate on it) | +| `api` (Dockerfile.backend) | **Deployment** (HPA target) + Service `:8000` | Runs **only** `uvicorn` (migrations moved to the Job); CPU `requests` set so HPA can compute % | +| `worker` (Dockerfile.backend) | **Deployment** (1 replica) | `python -m app.jobs.worker --loop`; same image/env; no ports | +| `frontend` (Dockerfile.frontend) | **Deployment** + Service `:3000` | `NEXT_PUBLIC_API_BASE_URL` is **build-time baked** (D11) | +| `prometheus` | **Deployment** + Service `:9090` + ConfigMap | `prometheus.yml`/`alerts.yml` from existing `deploy/prometheus/*` as a ConfigMap | +| `grafana` | **Deployment** + Service `:3000` + ConfigMap | provisioning + dashboard JSON as ConfigMaps; admin pw from Secret | +| (north-south) | **Ingress** (ingress-nginx) | host-based: `second-brain.local` → frontend, `api.second-brain.local` → api | +| (autoscaling) | **metrics-server** + **HPA** | HPA on api CPU; load via `hey`/`kubectl run` | + +**Tech delta:** no application-code changes. New top-level `deploy/k8s/` (manifests, kustomization, +`secret.example.yaml`), a new `.github/workflows/k8s.yml`, `docs/k8s-evidence/`, ADR-0014. The +existing eval-gated `ci.yml` is **untouched**. One additive, backwards-compatible change to +`deploy/Dockerfile.frontend` (a build `ARG`, see D11). `kind` + `kubectl` installed via winget if +missing (D1). + +## Decisions + +**D1–D10 are the goal's defaults (accepted).** **D11–D13 are decisions this plan adds** after +reading the stack — flagged here rather than discovered mid-build. + +- **D1 — Cluster = `kind`, multi-node** (1 control-plane + 2 workers) so HPA spread and + ingress-on-a-labelled-node are real, not single-node toys. Install `kind`+`kubectl` via winget if + absent (Docker Desktop/WSL2 already present). Cluster config carries `extraPortMappings` 80/443→host + and `ingress-ready=true` on the control-plane node. +- **D2 — Images built locally + `kind load docker-image`** (no registry, $0). Tag `…:phase7` (a + fixed non-`latest` tag) with **`imagePullPolicy: IfNotPresent`** so K8s uses the loaded image and + never tries a registry pull. +- **D3 — Postgres = StatefulSet + PVC** (`pgvector/pgvector:pg16`), reached in-cluster by Service DNS. + `alembic upgrade head` runs as a **migrate Job** (against the DB **directly**, mirroring the compose + `DIRECT_DATABASE_URL`), not inline in the api. api/worker wait on DB via an init wait + the Job is + applied and `kubectl wait`-ed before they roll. +- **D4 — Config split: ConfigMap (non-secret) + Secret (secret).** ConfigMap: in-cluster DSNs (Service + DNS), `SECOND_BRAIN_LLM_PROVIDER`, `NEXT_PUBLIC_API_BASE_URL`. Secret: DB password, Gemini API key, + admin token, Grafana admin password. **Secrets are never committed** — ship `secret.example.yaml` + only; the real Secret is created out-of-band (documented). +- **D5 — Ingress = ingress-nginx**, host-based to api + frontend; smoke `GET /health` through it. +- **D6 — Autoscaling = metrics-server + HPA on api (CPU target).** Demonstrate scale-up under load + (`hey`/`kubectl run … hey`), capture `kubectl get hpa` + pod count before/after as evidence. +- **D7 — Observability = reuse Prometheus + Grafana as simple Deployments**, configs via ConfigMap. + Trim (e.g. drop Grafana, keep Prometheus) **only** if local RAM forces it — and note the trim. +- **D8 — CI/CD = new GitHub Actions workflow** (`k8s.yml`, kind action): build+load images, install + ingress-nginx, apply manifests, `kubectl wait` rollouts, smoke `/health` through ingress, tear down. + The existing eval-gated `ci.yml` stays untouched and green. **HPA load demo stays a *local* evidence + step (D13)** — not in CI. +- **D9 — Managed cluster (GKE/EKS) = OPTIONAL, OFF BY DEFAULT.** No paid cloud resource without + flagging cost and waiting for an explicit OK (AGENTS.md cost rule). If ever approved → delete + immediately after. +- **D10 — Teardown:** after evidence is captured, `kind delete cluster`; nothing left running. + +- **D11 (ADDED) — `NEXT_PUBLIC_API_BASE_URL` is build-time baked; add a backwards-compatible build + `ARG` to `Dockerfile.frontend`.** Next.js inlines `NEXT_PUBLIC_*` at `next build`, so a runtime + ConfigMap value can't change what the browser fetches. The compose stack happens to pass it at + runtime (works only because compose's default points at `localhost:8000`). For K8s the browser must + call the API's **ingress** host, so the frontend image is **built with** + `NEXT_PUBLIC_API_BASE_URL=http://api.second-brain.local`. Implemented as `ARG + NEXT_PUBLIC_API_BASE_URL=http://localhost:8000` + `ENV` before `npm run build` — **additive and + default-preserving** (compose builds with no arg → unchanged behaviour). *Trade-off:* one Phase-6 + file touched, but only additively; recorded in implementation-notes. +- **D12 (ADDED) — pgbouncer configured by env, not a committed `userlist.txt`.** The compose stack + mounts `pgbouncer/userlist.txt`, which embeds the DB credential — committing that to a manifest/ + ConfigMap would leak a secret. `edoburu/pgbouncer` supports env config (`DB_HOST/DB_USER/ + DB_PASSWORD/POOL_MODE=session`) and auto-generates the userlist at start. So pgbouncer reads + password from the **Secret** via env. *Trade-off:* diverges from the file-mounted compose config, + but keeps the secret out of git and preserves session pooling (ADR-0012 — psycopg3 prepared + statements need session mode). +- **D13 (ADDED) — HPA scale-up evidence is captured *locally*, committed under `docs/k8s-evidence/`; + CI does build→load→apply→rollout→smoke→teardown only.** A load-driven autoscale is timing-sensitive + and the backend image carries torch (slow to build + RAM-heavy per replica); making CI assert "pods + went 1→N" would be flaky. CI proves the manifests *stand up and serve*; the autoscaling proof is the + local evidence artifact. *Trade-off:* CI doesn't gate on scaling — acceptable for a learning track; + the scaling is demonstrated + captured, just not in the pipeline. + +## File structure (created/modified in this phase) + +```text +deploy/ + Dockerfile.frontend # MODIFY (D11): additive ARG NEXT_PUBLIC_API_BASE_URL + k8s/ + kind-cluster.yaml # CREATE: multi-node kind config (extraPortMappings, ingress-ready) + namespace.yaml # CREATE: namespace second-brain + configmap.yaml # CREATE: non-secret config (DSNs, provider, api base url) + secret.example.yaml # CREATE: TEMPLATE only (no real secrets committed) + postgres-statefulset.yaml # CREATE: StatefulSet + headless Service + PVC + migrate-job.yaml # CREATE: Job alembic upgrade head (direct DSN) + pgbouncer.yaml # CREATE: Deployment + Service :6432 (env-configured, D12) + redis.yaml # CREATE: Deployment + Service :6379 + api.yaml # CREATE: Deployment (CPU requests) + Service :8000 + worker.yaml # CREATE: Deployment (--loop) + frontend.yaml # CREATE: Deployment + Service :3000 + ingress.yaml # CREATE: host-based ingress → frontend + api + api-hpa.yaml # CREATE: HorizontalPodAutoscaler (api, CPU) + monitoring/ + prometheus.yaml # CREATE: Deployment + Service + ConfigMap (from deploy/prometheus/*) + grafana.yaml # CREATE: Deployment + Service + ConfigMaps (provisioning + dashboard) + kustomization.yaml # CREATE: orders core resources for `kubectl apply -k` + README.md # CREATE: apply order + run/verify + teardown +.github/workflows/ + k8s.yml # CREATE: kind CI (build+load+apply+rollout+smoke+teardown) +docs/ + phase-7-plan.md # THIS FILE + adr/0014-kubernetes-learning-track.md # CREATE + adr/README.md # MODIFY: index 0014 + k8s-evidence/ # CREATE: captured kubectl output (rollout, ingress smoke, HPA scale) + PROGRESS.md implementation-notes.md # MODIFY: phase-7 → complete + off-spec notes +README.md / backend README # MODIFY: "Phase 7 — run & verify" section +``` + +## Tasks (apply → verify gate) + +> Branch `phase-7-impl` off `main`. Commit `docs/phase-7-plan.md` **first**. Then one commit per green +> layer. Windows Git-Bash junk-file quirk: `git clean -f` before each commit, explicit `git add`. + +0. **Plan + tooling.** Commit this plan. Verify/install `kind`+`kubectl` (winget). **Gate:** `kind + version`, `kubectl version --client`, `docker version` all succeed. Commit `docs: phase-7 plan + (K8s learning track) + D1–D13`. +1. **Cluster + namespace.** `kind-cluster.yaml` (multi-node, ingress-ready, port maps); create + cluster; `namespace.yaml`. **Gate:** `kubectl get nodes` shows control-plane + 2 workers `Ready`; + `kubectl get ns second-brain` exists. Commit `feat(k8s): multi-node kind cluster config + namespace`. +2. **Config + secrets.** `configmap.yaml` (Service-DNS DSNs, provider, api base url) + + `secret.example.yaml` (template). Create the **real** Secret locally from the template (NOT + committed). **Gate:** `kubectl get configmap second-brain-config` + `kubectl get secret + second-brain-secrets` present; `git status` shows no real secret staged. Commit `feat(k8s): + configmap + secret template (secrets uncommitted, D4)`. +3. **Postgres StatefulSet + migrate Job.** Build backend image + `kind load`. Apply + `postgres-statefulset.yaml`; once Ready, apply `migrate-job.yaml`. **Gate:** pg pod `Ready` + + `kubectl exec … pg_isready` ok; `kubectl wait --for=condition=complete job/migrate` and its logs + show `alembic upgrade head` → a revision (`0004…`). Commit `feat(k8s): postgres statefulset + PVC + + migrate job`. +4. **pgbouncer + redis.** Apply both. **Gate:** both pods `Ready`; `kubectl exec` a `psql` through + `pgbouncer:6432` returns `SELECT 1`; redis `PING`→`PONG`. Commit `feat(k8s): pgbouncer (env-config, + D12) + redis`. +5. **api + worker.** `kind load` (reuse image). Apply `api.yaml` (CPU requests) + `worker.yaml`. + **Gate:** `kubectl rollout status deploy/api` + `deploy/worker` complete; `kubectl port-forward + svc/api 8000` → `curl /health` 200; worker logs show "no eligible job" (clean idle). Commit + `feat(k8s): api + worker deployments`. +6. **frontend.** Build frontend image **with the ingress api host baked** (D11) + `kind load`. Apply + `frontend.yaml`. **Gate:** `kubectl rollout status deploy/frontend`; port-forward → `curl /` serves + HTML. Commit `feat(k8s): frontend deployment (NEXT_PUBLIC baked, D11)`. +7. **Ingress.** Install ingress-nginx (kind provider manifest), wait for the controller; apply + `ingress.yaml`. **Gate:** `curl -H 'Host: api.second-brain.local' http://localhost/health` → 200; + `curl -H 'Host: second-brain.local' http://localhost/` → frontend HTML. Capture both to evidence. + Commit `feat(k8s): ingress-nginx host-based routing to api + frontend`. +8. **metrics-server + HPA + load demo.** Install metrics-server (`--kubelet-insecure-tls` for kind); + apply `api-hpa.yaml`. Drive load (`hey` against `/health` through the api Service). **Gate:** + `kubectl top pods` returns metrics; `kubectl get hpa` shows CPU% climbing; api replicas scale + above `minReplicas`. Capture `get hpa` + `get pods` before/under/after load → evidence. Commit + `feat(k8s): metrics-server + api HPA + load-scale evidence`. +9. **Monitoring (Prometheus + Grafana).** ConfigMaps from `deploy/prometheus/*` + `deploy/grafana/*`; + apply `monitoring/`. **Gate:** both pods `Ready`; Prometheus `/-/healthy` 200 and its api target is + `up`; Grafana `/api/health` 200. Trim per D7 if RAM-bound (note it). Commit `feat(k8s): prometheus + + grafana deployments (reused configs)`. +10. **CI/CD (`k8s.yml`).** kind-action workflow: build+load both images, install ingress-nginx, apply + manifests, `kubectl wait` rollouts, smoke `/health` via ingress, then teardown. Inject dummy + secrets via workflow env (no real keys). **Gate:** YAML parses (`actionlint`/local), and the + workflow is **green on the PR** (alongside the untouched `ci.yml`). Commit `ci(k8s): kind + build→apply→rollout→smoke→teardown workflow`. +11. **Teardown + docs.** `kind delete cluster`; confirm nothing runs. ADR-0014 + index; flip PROGRESS + Phase 7 → ✅ (dated); off-spec notes (D11/D12/D13, any trims) in implementation-notes; README + "Phase 7 — run & verify". **Gate:** `kind get clusters` empty / `docker ps` shows no kind/stack + containers; docs updated. Commit `docs: phase-7 ADR-0014 + evidence + run/verify + progress; + teardown`. + +## Self-review (against AGENTS.md Phase 7 + the 8 services) + +- Real manifests for all 8 compose services → Tasks 3–9 ✅ (db, pgbouncer, redis, api, worker, frontend, prometheus, grafana) +- StatefulSet + PVC for Postgres, migrations as a Job → Task 3 ✅ +- Host-based ingress, /health smoked through it → Task 7 ✅ +- HPA autoscaling demonstrated under load with captured evidence → Task 8 ✅ +- Prometheus + Grafana reused → Task 9 ✅ +- New K8s CI/CD workflow, existing eval CI untouched + green → Task 10 ✅ +- Secrets not committed (template only) → Task 2 / D4 ✅ +- Cluster torn down, $0, nothing left running → Task 11 / D10 ✅ +- No paid cloud without explicit OK → D9 (off by default) ✅ +- Docs: ADR-0014, PROGRESS dated, implementation-notes, README run/verify → Task 11 ✅ + +## Known sharp edges (flagged objections — not placeholders) + +1. **Next.js `NEXT_PUBLIC_*` is build-time, not runtime (D11).** The browser→API call needs the + ingress host baked at `next build`. Mitigation: additive build `ARG` on `Dockerfile.frontend`; + smoke proves `/health` (api via ingress) **and** UI HTML served. Browser-driven UI→API is exercised + via the baked host. +2. **kind never pulls loaded images.** Must use a fixed non-`latest` tag + `imagePullPolicy: + IfNotPresent`, else K8s tries (and fails) a registry pull → `ImagePullBackOff`. (D2.) +3. **HPA needs CPU `requests` on api**, and metrics-server on kind needs `--kubelet-insecure-tls`. + Without requests the HPA shows `` and never scales. +4. **RAM on a laptop.** The backend image loads MiniLM/torch per replica; multiple api replicas + + Prometheus + Grafana can pressure WSL2 memory. Mitigations: modest `requests`, HPA `maxReplicas` + capped (≤4), drive the HPA with `/health` (no embedder load), trim monitoring per D7 if needed. +5. **Migrations are a Job, so api/worker must NOT migrate.** The api compose `command` prefixes + `alembic upgrade head`; the K8s api runs *only* uvicorn. The Job runs once against the DB directly; + api/worker tolerate a brief pre-migration race by restarting (same posture as the compose worker). +6. **pgbouncer secret handling (D12).** Use env-based config so the DB password comes from the Secret, + never a committed `userlist.txt`. Session pool mode preserved (psycopg3 prepared statements). +7. **CI build time.** Building the torch-bearing backend image in CI is minutes-slow; acceptable, but + the HPA load demo stays local (D13) to keep CI deterministic. +8. **Windows specifics.** Smoke uses `curl -H 'Host: …' http://localhost/…` (kind port-map) so no + admin `hosts` edit is needed; the real `hosts` entry (`127.0.0.1 second-brain.local + api.second-brain.local`) is documented for browser use. `git clean -f` + explicit `git add` guard + the Git-Bash junk-file quirk.