diff --git a/.github/workflows/k8s.yml b/.github/workflows/k8s.yml index e033bcb..266c65a 100644 --- a/.github/workflows/k8s.yml +++ b/.github/workflows/k8s.yml @@ -88,7 +88,15 @@ jobs: -l app.kubernetes.io/component=controller --timeout=180s - name: Apply manifests - run: kubectl apply -k deploy/k8s + run: | + # ingress-nginx can mark the controller pod Ready before the admission webhook + # accepts HTTPS. Retry the apply so the Ingress resource survives that CI race. + for i in 1 2 3 4 5; do + kubectl apply -k deploy/k8s && exit 0 + echo "kubectl apply failed on attempt $i; waiting for admission webhook" + sleep 15 + done + kubectl apply -k deploy/k8s - name: Wait for rollouts run: | diff --git a/.gitignore b/.gitignore index 4bb857d..d4672e3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ backend/.env # Phase 6 production secrets — only the *.example templates are committed deploy/.env.prod deploy/pgbouncer/userlist.txt +# Box-specific compose override (carries the public host/IP) — commit the .example only +deploy/docker-compose.vps.yml # Phase 7 K8s secret — only secret.example.yaml is committed (D4) deploy/k8s/secret.yaml diff --git a/deploy/.env.prod.example b/deploy/.env.prod.example index 98ee03d..3f2e014 100644 --- a/deploy/.env.prod.example +++ b/deploy/.env.prod.example @@ -1,5 +1,5 @@ # Production env template (Phase 6). Copy to deploy/.env.prod (gitignored) and fill in. -# Used by: docker compose -f deploy/docker-compose.prod.yml --env-file deploy/.env.prod up -d +# Used by: docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod up -d # --- Postgres --- POSTGRES_USER=second_brain diff --git a/deploy/caddy/Caddyfile b/deploy/caddy/Caddyfile new file mode 100644 index 0000000..ae58757 --- /dev/null +++ b/deploy/caddy/Caddyfile @@ -0,0 +1,21 @@ +# Caddy reverse proxy for the single-VPS prod stack. +# One public origin, auto-HTTPS (Let's Encrypt). The site address is injected from the +# environment so this file stays host-agnostic: +# CADDY_SITE_ADDRESS=YOUR_VPS_IP.sslip.io (sslip.io maps .sslip.io -> , +# so Caddy can get a publicly-trusted cert without owning a domain). +# Swap in a real domain later by changing CADDY_SITE_ADDRESS — no other change needed. +{$CADDY_SITE_ADDRESS} { + encode zstd gzip + + # API: strip the /api prefix and forward to the FastAPI service. + # https:///api/chat -> api:8000/chat + # https:///api/health -> api:8000/health + handle_path /api/* { + reverse_proxy api:8000 + } + + # Everything else: the Next.js frontend (served same-origin, so no CORS). + handle { + reverse_proxy frontend:3000 + } +} diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index f5c2266..bbcba00 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -1,6 +1,6 @@ # Production stack for the single VPS (Phase 6, ADR-0011/0012). NOT run in CI and NOT the # dev compose (that one is db-only). Bring up with: -# docker compose -f deploy/docker-compose.prod.yml --env-file deploy/.env.prod up -d --build +# docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod up -d --build # Everything is one Docker Compose stack on one box, per AGENTS.md. services: db: diff --git a/deploy/docker-compose.vps.yml.example b/deploy/docker-compose.vps.yml.example new file mode 100644 index 0000000..08e4af6 --- /dev/null +++ b/deploy/docker-compose.vps.yml.example @@ -0,0 +1,59 @@ +# VPS-specific override TEMPLATE for the single-box prod stack. +# Copy to deploy/docker-compose.vps.yml (gitignored — it carries your public host/IP) and +# replace YOUR_VPS_IP with the droplet's public IP, then deploy: +# docker compose -p second-brain \ +# -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml \ +# --env-file deploy/.env.prod up -d +# +# What it does on top of docker-compose.prod.yml: +# - bakes the public API base URL into the Next.js bundle at BUILD time (NEXT_PUBLIC_* is +# inlined by `next build`; a runtime env var is ignored by the browser). The browser calls +# the API through Caddy at https:///api, same-origin. +# - allows that origin in the API's CORS list (belt-and-suspenders; same-origin needs none). +# - keeps Prometheus/Grafana bound to localhost only (reach them via SSH tunnel). +# - adds Caddy as the public entrypoint on 80/443 with auto-HTTPS. +# Using .sslip.io gives Caddy a real Let's Encrypt cert without owning a domain +# (sslip.io resolves .sslip.io -> ). Swap in a real domain by changing these values. +services: + frontend: + build: + args: + NEXT_PUBLIC_API_BASE_URL: https://YOUR_VPS_IP.sslip.io/api + ports: !override + - "127.0.0.1:3000:3000" + + api: + environment: + SECOND_BRAIN_CORS_ORIGINS: '["https://YOUR_VPS_IP.sslip.io"]' + ports: !override + - "127.0.0.1:8000:8000" + + # !override REPLACES the base file's port list (compose concatenates by default, which would + # bind 0.0.0.0:PORT and 127.0.0.1:PORT both -> "address already in use"). Localhost-only here. + prometheus: + ports: !override + - "127.0.0.1:9090:9090" + + grafana: + ports: !override + - "127.0.0.1:3001:3000" + + caddy: + image: caddy:2-alpine + depends_on: + - api + - frontend + ports: + - "80:80" + - "443:443" + environment: + CADDY_SITE_ADDRESS: ${CADDY_SITE_ADDRESS:-YOUR_VPS_IP.sslip.io} + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + restart: unless-stopped + +volumes: + caddy_data: + caddy_config: diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 08e7fff..5b77fd7 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -23,6 +23,38 @@ Legend: ⬜ not started · 🟡 in progress · ✅ complete Add a dated entry per working session. Most recent on top. +### 2026-06-02 — LIVE on the VPS: full stack up + Caddy HTTPS, end-to-end verified +- **What:** brought the production stack fully live on the **DigitalOcean droplet** + (`YOUR_VPS_IP`, 2 GB, project **`second-brain`**, files `docker-compose.prod.yml` + + `docker-compose.vps.yml`, embeddings offloaded to Gemini so it fits 2 GB). Added a **Caddy** + reverse proxy with **real Let's Encrypt HTTPS** via the no-domain host + **`YOUR_VPS_IP.sslip.io`** (`/api/*` → api, everything else → frontend; http→https 308). + New files: `deploy/caddy/Caddyfile`, `deploy/docker-compose.vps.yml`, `docs/USAGE.md`. +- **Found the API live but only 4/8 services up** (api, db, pgbouncer, redis); frontend, worker, + grafana, prometheus stuck in **Created**. Root cause: the `vps.yml` override **added** a + `127.0.0.1:` port to prometheus/grafana, and **compose concatenates port lists** rather than + replacing — so each tried to bind both `0.0.0.0:PORT` and `127.0.0.1:PORT` → "address already + in use" (exit 128), which aborted the `up` and left the rest Created. **Fix:** `ports: !override` + in the override (compose v2.24+ tag) so the localhost binding replaces the base one. +- **Two more fixes:** (1) the frontend image baked `NEXT_PUBLIC_API_BASE_URL` at **build** time + but the override only passed it as `build.args` (correct) — rebuilt with the HTTPS `/api` URL so + the browser bundle isn't mixed-content; (2) documented the **project-name gotcha**: omitting + `-p second-brain` resolves the project to `deploy` and spins up an empty duplicate (hit it once, + cleaned it up incl. orphan volumes). +- **Verified end-to-end over HTTPS** (real cert, from off-box): `/api/health` ok; `/ingest` + (`type:manual` — `note` violates `sources_type_check`) embedded via Gemini; `/search` retrieved + it; `/chat` returned a **cited** `gemini-2.5-flash` answer (1.4 s); `/briefing` produced a + Gemini briefing after enqueue. All 8 app services + caddy **Up**; cert auto-renews (exp 2026-08-31). +- **Ops wired:** daily briefing **cron** installed at `/etc/cron.d/second-brain-briefing` (07:00, + correct `-p second-brain` invocation — the runbook's old line would've hit the project-name bug). + Grafana/Prometheus stay bound to `127.0.0.1` (SSH-tunnel only). PR #14 follow-up also made the + VPS override bind direct API/frontend ports to `127.0.0.1`, leaving Caddy 80/443 as the public surface. +- **PR #14 verification fix:** the fresh head reproduced the Phase-7 `kind-smoke` ingress-nginx + admission webhook race twice (`connect: connection refused` during `kubectl apply -k`). Patched + `.github/workflows/k8s.yml` to retry the apply through that transient readiness gap. +- **Known follow-ups:** briefing `body_markdown` has mojibake em-dash/middot (cosmetic, app-code, + spawned as a separate task); enable `ufw` (USAGE.md §hardening). + ### 2026-06-02 — Live deploy validated LOCALLY on Docker Desktop (prod Compose stack) + Gemini model fix - **What:** brought up `deploy/docker-compose.prod.yml` end-to-end on Docker Desktop (project `second-brain-prod`, isolated from the dev DB on 5433) with a real Gemini API key — running the diff --git a/docs/USAGE.md b/docs/USAGE.md new file mode 100644 index 0000000..4e8788e --- /dev/null +++ b/docs/USAGE.md @@ -0,0 +1,206 @@ +# Second Brain — Usage Guide + +How to use and operate the live deployment. Last verified **2026-06-02** against the +production droplet. + +--- + +## Live URLs + +> Replace `YOUR_VPS_IP` in the URLs below with your droplet's public IP address. + +| What | URL | Notes | +|---|---|---| +| **Web UI** | **https://YOUR_VPS_IP.sslip.io** | Chat + search. Redirects to `/chat`. | +| **API (app path)** | https://YOUR_VPS_IP.sslip.io/api | Behind Caddy, same TLS cert. e.g. `/api/health`, `/api/chat`. | +| **API (direct)** | http://localhost:8000 *(on the box or via SSH tunnel)* | Plain HTTP, bound to localhost only. Handy for quick `curl`. | +| **Swagger UI** | http://localhost:8000/docs *(on the box or via SSH tunnel)* | Interactive "try it" docs for every endpoint. | +| Grafana | http://localhost:3001 *(via SSH tunnel)* | admin / `GRAFANA_ADMIN_PASSWORD`. Not public. | +| Prometheus | http://localhost:9090 *(via SSH tunnel)* | Not public. | + +> **TLS:** `YOUR_VPS_IP.sslip.io` is a wildcard-DNS hostname that resolves to the droplet's +> IP (`sslip.io` maps `.sslip.io → `), which lets Caddy obtain a real, auto-renewing +> Let's Encrypt certificate **without owning a domain**. `http://` auto-redirects to `https://`. +> To switch to a real domain later: point an A record at the IP, set `CADDY_SITE_ADDRESS` in +> `deploy/.env.prod` and the frontend build arg in `deploy/docker-compose.vps.yml`, then rebuild +> the frontend + restart Caddy. + +--- + +## What it is + +Second Brain is a personal RAG assistant: you **ingest** notes/text, it embeds and stores them +in Postgres + pgvector, and you **chat** or **search** over them with **cited** answers. It also +produces a **daily briefing** and exposes **agentic tools** over MCP. The LLM +(`gemini-2.5-flash`) and embeddings (`gemini-embedding-001`) are hosted Gemini API calls, so the +box needs no GPU and fits in 2 GB RAM. + +**Architecture:** one Docker Compose project (`second-brain`) on one DigitalOcean droplet, 9 +services: `caddy` (HTTPS reverse proxy) → `frontend` (Next.js) + `api` (FastAPI); `worker` +(daily briefing + async research); `db` (pgvector), `pgbouncer`, `redis`; `prometheus` + +`grafana`. + +--- + +## Using the Web UI + +Open **https://YOUR_VPS_IP.sslip.io**. You get: + +- **/chat** — ask a question; the answer comes back with inline `[1]`,`[2]` citation markers. + Click a marker to see the source card (title, snippet, score). A conversation sidebar lists + past threads (auto-refresh). Thumbs up/down records feedback. A "private mode" toggle routes + that turn through the local LLM path instead of Gemini (if configured). +- **/search** — raw hybrid (vector + full-text) search results with source/tag filters, no LLM. + +The browser talks to the API at `…/api` through Caddy (same origin, so no CORS issues). + +--- + +## Using the API + +Base URL `https://YOUR_VPS_IP.sslip.io/api`. Examples use `curl` (works on Windows 11 and +the box). + +### Add notes — `POST /ingest` +`source.type` must be one of: **`manual`**, `notes_folder`, `github`, `rss`, `pdf_upload`, +`bookmark`, `research_note`. Use `manual` for ad-hoc text. + +```bash +curl -X POST https://YOUR_VPS_IP.sslip.io/api/ingest \ + -H "Content-Type: application/json" -d '{ + "source": {"type": "manual", "name": "My Notes"}, + "documents": [ + {"title": "HNSW tuning", "content": "m=16, ef_construction=64, cosine distance.", + "tags": ["postgres", "vector"]} + ] + }' +``` +Re-ingesting identical content is deduped by content hash (`status: "duplicate"`). + +### Ask — `POST /chat` +```bash +curl -X POST https://YOUR_VPS_IP.sslip.io/api/chat \ + -H "Content-Type: application/json" \ + -d '{"message": "How should I tune the HNSW index?"}' +``` +Returns `answer` (with `[n]` markers), `citations[]`, token `usage`, `model`, `latency_ms`, and +`conversation_id`. Pass `conversation_id` back to continue a thread. Options: +`{"message":"…","top_k":8,"filters":{"tags":["postgres"]},"options":{"private_mode":false}}`. +If nothing relevant is found it refuses rather than inventing an answer. + +### Search — `GET /search` +```bash +curl "https://YOUR_VPS_IP.sslip.io/api/search?q=hnsw+tuning&top_k=5" +``` + +### Other endpoints +- `GET /briefing`, `GET /briefing/history` — daily briefings (see below). +- `GET /conversations`, `GET /conversations/{id}` — chat history with reconstructed citations. +- `POST /feedback` — `{"message_id": 123, "rating": 1}` (rating is `1` or `-1`). +- `GET /health` — `{"status":"ok","db":"ok","embedder":"…"}`. + +--- + +## Daily briefing + +The `worker` service drains a job queue continuously; a host cron enqueues one `briefing` job a +day. It's already installed at **`/etc/cron.d/second-brain-briefing`** (07:00 server time). Read +it the next morning at `GET /api/briefing`. Each run summarizes documents ingested since the +previous briefing. Enqueue one on demand: + +```bash +cd /root/second-brain +docker compose -p second-brain \ + -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml \ + --env-file deploy/.env.prod exec -T worker python -m app.jobs.enqueue briefing +``` + +--- + +## Agentic tools (MCP) + +The MCP server (`backend/app/mcp_server.py`, stdio) exposes five tools: `search_notes`, +`create_task`, `list_tasks`, `send_digest`, and `research_topic` (the LLM researches a topic, +stores it as a note, and auto-indexes it so it's permanently searchable). Wire it into a local +MCP client (e.g. Claude Desktop) — run it on the box or locally with the DB DSN + Gemini key in +its `env`. Set `SECOND_BRAIN_LLM_PROVIDER=fake` for a keyless smoke test. + +--- + +## Operating the box + +```bash +ssh root@YOUR_VPS_IP +cd /root/second-brain + +# the stack is ONE project; always pass -p second-brain + BOTH compose files + the env file: +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" + +$DC ps # status of all 9 services +$DC logs -f api worker # follow logs +$DC restart api # restart one service +$DC up -d # reconcile / start everything +$DC down # stop the whole stack +``` + +> **Gotcha:** the project name is **`second-brain`** and the stack is composed of **two** files. +> Running `docker compose -f deploy/docker-compose.prod.yml …` *without* `-p second-brain` (and +> without the `vps.yml` override) resolves to a **different project** (`deploy`) and will spin up +> an empty duplicate. Always use the `$DC` invocation above. + +**Update to a new version** (deploy only commits whose CI/eval-gate is green): +```bash +git pull +$DC up -d --build # rebuilds changed images, applies migrations, restarts +curl -s localhost:8000/health +``` +Changing the frontend's API URL or the Caddy host requires `--build frontend` (the API base URL +is baked into the bundle at build time). + +**Backup before any migration:** +```bash +$DC exec -T db pg_dump -U second_brain second_brain | gzip > backup-$(date +%F).sql.gz +``` + +**Monitoring (kept private — reach via SSH tunnel from your laptop):** +```bash +ssh -L 3001:localhost:3001 -L 9090:localhost:9090 root@YOUR_VPS_IP +# then open http://localhost:3001 (Grafana) and http://localhost:9090 (Prometheus) +``` + +--- + +## Admin / data-ops + +`SECOND_BRAIN_ADMIN_TOKEN` is set, so the governed endpoints are **enabled** and require a +bearer token: +- `GET /api/data/export?source_id=…` — export a source (GDPR access). +- `DELETE /api/data/sources/{id}` — delete a source + its documents (GDPR erasure). +- `POST /api/admin/retention/purge` — null `raw_text` past the retention TTL. + +```bash +curl -H "Authorization: Bearer " \ + "https://YOUR_VPS_IP.sslip.io/api/data/export?source_id=3" +``` + +--- + +## Security notes / hardening backlog + +The deployment is functional and uses real HTTPS, but a few things are worth tightening: + +1. **Enable a host firewall.** `ufw` is currently inactive; allow only 22/80/443. +2. **Privacy:** with `SECOND_BRAIN_EMBEDDING_PROVIDER=gemini`, note text is sent to Google at + **ingest** (not just chat). Switch to `local` embeddings for a fully private path (needs a + ≥4 GB box for the torch model). + +--- + +## Run locally (dev) + +```bash +docker compose up -d db # dev compose is db-only (host port 5433) +cd backend && alembic upgrade head && uvicorn app.main:app --reload # :8000 +cd frontend && npm run dev # :3000 +``` +Set `SECOND_BRAIN_GEMINI_API_KEY` (or `SECOND_BRAIN_LLM_PROVIDER=fake` to run without a key). diff --git a/docs/adr/0011-vps-provider.md b/docs/adr/0011-vps-provider.md index b365f05..9a28b0d 100644 --- a/docs/adr/0011-vps-provider.md +++ b/docs/adr/0011-vps-provider.md @@ -5,6 +5,14 @@ - **Deciders:** project owner (low cost is the stated priority) - **Context phase:** Phase 6 (productionize on a VPS) +> **Update 2026-06-02 — supersedes the region & fallback below.** The owner is in the **USA**, +> not SEA — the "SEA (Vietnam)" premise in Context #2 and the Singapore region choice are +> incorrect. **Corrected:** Oracle Cloud Always Free, home region **US Central (Chicago, +> `us-chicago-1`)** ($0, up to 4 ARM OCPU / 24 GB). Paid fallback flips to **Hetzner US** +> (Ashburn VA / Hillsboro OR, ~$5/mo, x86, no ARM-capacity lottery) — Hetzner was dismissed +> below *only* for Vietnam latency, which no longer applies. The RAM-constraint analysis, +> multi-arch note, and $0-baseline rationale all still stand. + ## Context The whole stack runs as one Docker Compose project on a single always-on box (AGENTS.md). diff --git a/docs/implementation-notes.md b/docs/implementation-notes.md index 0847433..d7a24f9 100644 --- a/docs/implementation-notes.md +++ b/docs/implementation-notes.md @@ -9,6 +9,61 @@ what I gave up**. Keep it honest — the surprises are the valuable part. --- +## Going live on the VPS — Caddy HTTPS + compose override gotchas (2026-06-02) + +### Caddy reverse proxy with no-domain HTTPS via `sslip.io` +- **What:** added `deploy/caddy/Caddyfile` + a `caddy` service (in `deploy/docker-compose.vps.yml`) + fronting the stack on 80/443. `handle_path /api/*` strips the prefix to `api:8000`; everything + else proxies to `frontend:3000`. Site address is env-injected (`{$CADDY_SITE_ADDRESS}`). +- **Why:** the owner has no domain, but the UI needs HTTPS (and the browser bundle calling the API + over plain HTTP from an HTTPS page is blocked as mixed content). `sslip.io` resolves + `YOUR_VPS_IP.sslip.io → YOUR_VPS_IP`, so Caddy gets a **real, auto-renewing Let's + Encrypt cert** with zero DNS setup. Same-origin proxying also makes CORS a non-issue. Verified: + valid cert (exp 2026-08-31), http→https 308, `/api/health` + UI + ingest/search/chat all 200. +- **Trade-off:** the sslip host is baked into the frontend bundle + the override (not portable); + swapping to a real domain later = change `CADDY_SITE_ADDRESS` + the frontend build arg + rebuild. + Acceptable. Moving to a real domain is a one-liner change away. + +### Compose **concatenates** port lists across files → use `!override` +- **What / why:** the base `docker-compose.prod.yml` publishes prometheus `9090:9090` and grafana + `3001:3000` on `0.0.0.0`; the VPS override wanted them on `127.0.0.1` only. Adding + `127.0.0.1:9090:9090` in the override **appended** to (didn't replace) the base entry, so the + container tried to bind **both** `0.0.0.0:9090` and `127.0.0.1:9090` → second bind fails + "address already in use" (exit 128). This aborted the whole `up`, leaving 4 services in + **Created** (the symptom: API live but UI/worker/monitoring down). **Fix:** `ports: !override` + (compose v2.24+) in the override so the localhost binding replaces the base list. +- **Trade-off:** none — strictly correct. The lesson generalizes: compose merges **maps** by key + but **concatenates sequences**; to replace a sequence you need `!override`/`!reset`. +- **PR #14 hardening follow-up:** extended the same `!override` pattern to `api:8000` and + `frontend:3000`, binding both direct HTTP ports to `127.0.0.1`. Caddy 80/443 is now the only + public surface by default; direct API/Swagger access is still available on the box or through an + SSH tunnel. + +### Project-name gotcha — always `-p second-brain` +- **What:** running `docker compose -f deploy/docker-compose.prod.yml …` from the repo root without + `-p` resolves the project name to **`deploy`** (the compose file's parent dir), creating an empty + **duplicate** project (separate volumes → empty DB). The real stack is project `second-brain`. +- **How to apply:** every prod compose command must use + `-p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod`. + The deploy-checklist + the briefing cron were corrected accordingly; documented in `docs/USAGE.md`. + +### CI ingress-nginx admission race +- **What:** PR #14 head checks reproduced the Phase-7 `kind-smoke` flake twice: `kubectl apply -k` + reached the Ingress while the ingress-nginx admission service existed but its HTTPS endpoint still + returned `connect: connection refused`. +- **Fix:** changed `.github/workflows/k8s.yml` to retry `kubectl apply -k deploy/k8s` with short + sleeps. The first apply may create all non-Ingress resources, and a later retry creates/updates + the Ingress once the webhook is actually serving. +- **Trade-off:** the workflow can spend up to about a minute longer on this step, but it avoids + hiding real rollout failures because the later rollout/status/smoke steps still fail normally. + +### `sources.type` is constrained +- The `sources_type_check` constraint allows only `notes_folder | github | rss | pdf_upload | + bookmark | research_note | manual`. Ad-hoc ingest must use **`manual`** (the value all tests use); + `note` 500s with a CheckViolation. Reflected in the USAGE.md examples. + +--- + ## Hosted Gemini embeddings provider — fit the box on a 2 GB VPS (2026-06-02) ### `embedding_provider=gemini` drops the local torch/MiniLM footprint diff --git a/docs/runbooks/backup-restore.md b/docs/runbooks/backup-restore.md index 50e0e72..40287d4 100644 --- a/docs/runbooks/backup-restore.md +++ b/docs/runbooks/backup-restore.md @@ -14,9 +14,10 @@ data, so back them up). Redis is a cache (disposable). MLflow `./mlruns` is rege ```bash # /etc/cron.daily/second-brain-backup (chmod +x) set -euo pipefail -cd /home/USER/second-brain +cd /root/second-brain +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" ts=$(date +%Y%m%d-%H%M%S) -docker compose -f deploy/docker-compose.prod.yml exec -T db \ +$DC exec -T db \ pg_dump -U second_brain -d second_brain -Fc \ > /var/backups/second-brain/sb-$ts.dump # keep 14 days @@ -28,11 +29,13 @@ non-empty (`ls -la`) and periodically test-restore it (below) — an untested ba ## Restore (full) ```bash # into a clean database (DANGER: drops existing objects) -docker compose -f deploy/docker-compose.prod.yml up -d db -cat sb-YYYYMMDD-HHMMSS.dump | docker compose -f deploy/docker-compose.prod.yml exec -T db \ +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" + +$DC up -d db +cat sb-YYYYMMDD-HHMMSS.dump | $DC exec -T db \ pg_restore -U second_brain -d second_brain --clean --if-exists --no-owner # pgvector extension + RLS policies come from the dump; confirm: -docker compose -f deploy/docker-compose.prod.yml exec db \ +$DC exec db \ psql -U second_brain -d second_brain -c "SELECT count(*) FROM embeddings;" ``` @@ -40,16 +43,20 @@ docker compose -f deploy/docker-compose.prod.yml exec db \ Restore the latest dump into a throwaway database and run a sanity query — proves the backup is actually recoverable: ```bash -docker compose -f deploy/docker-compose.prod.yml exec db createdb -U second_brain sb_restore_test -cat sb-latest.dump | docker compose ... exec -T db pg_restore -U second_brain -d sb_restore_test --no-owner -docker compose ... exec db psql -U second_brain -d sb_restore_test -c "\dt" -docker compose ... exec db dropdb -U second_brain sb_restore_test +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" + +$DC exec db createdb -U second_brain sb_restore_test +cat sb-latest.dump | $DC exec -T db pg_restore -U second_brain -d sb_restore_test --no-owner +$DC exec db psql -U second_brain -d sb_restore_test -c "\dt" +$DC exec db dropdb -U second_brain sb_restore_test ``` ## Before a migration release Always snapshot first, so a bad migration is recoverable: ```bash -docker compose ... exec -T db pg_dump -U second_brain -d second_brain -Fc > pre-migrate-$(date +%s).dump +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" + +$DC exec -T db pg_dump -U second_brain -d second_brain -Fc > pre-migrate-$(date +%s).dump # deploy; if the migration misbehaves: alembic downgrade -1 (or restore the dump) ``` diff --git a/docs/runbooks/deploy-checklist.md b/docs/runbooks/deploy-checklist.md index dfbb8d9..0adb614 100644 --- a/docs/runbooks/deploy-checklist.md +++ b/docs/runbooks/deploy-checklist.md @@ -3,11 +3,18 @@ The live deploy (deferred from Phase 6 until the box is provisioned per ADR-0011). Everything runs as one Docker Compose stack on one VPS. Eval-gated: only deploy a commit whose CI is green. -## 0. Provision the box (ADR-0011) -- **Primary:** Oracle Cloud Always Free, **Singapore** region, VM.Standard.A1.Flex, 4 OCPU / - 24 GB / ~100 GB boot. ARM64 — our images are multi-arch. -- **Fallback:** Contabo Cloud VPS 10 (Singapore), 8 GB. -- Open only the ports you serve (see step 5). Add a swapfile if on 4 GB. +> **Status (2026-06-02): LIVE** on a DigitalOcean droplet (`YOUR_VPS_IP`), project +> `second-brain`, with a Caddy HTTPS reverse proxy (adds `deploy/docker-compose.vps.yml` + +> `deploy/caddy/Caddyfile` on top of the base). To **operate the running box**, see +> **`docs/USAGE.md`**; this runbook stays the from-scratch provisioning reference. Every prod +> compose command must use: +> `docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod …` + +## 0. Provision the box (ADR-0011, amended 2026-06-02 — US-based owner) +- **Primary:** Oracle Cloud Always Free, **US Central (Chicago, `us-chicago-1`)** home region, + VM.Standard.A1.Flex, up to 4 OCPU / 24 GB / ~100 GB boot. ARM64 — our images are multi-arch. +- **Fallback:** Hetzner US (Ashburn VA / Hillsboro OR), ~$5/mo, x86 — instant, no ARM-capacity lottery. +- Open only the ports you serve (see step 5). Add a swapfile if on a small (≤4 GB) box. ## 1. Install Docker + Compose ```bash @@ -35,10 +42,12 @@ cd backend && python -m app.eval.gate # exit 0 = quality OK ## 4. Bring up the DB first, then generate the PgBouncer userlist ```bash -docker compose -f deploy/docker-compose.prod.yml --env-file deploy/.env.prod up -d db +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" + +$DC up -d db # wait for healthy, then copy the SCRAM verifier into the (gitignored) userlist: cp deploy/pgbouncer/userlist.txt.example deploy/pgbouncer/userlist.txt -docker compose -f deploy/docker-compose.prod.yml exec db \ +$DC exec db \ psql -U second_brain -tAc \ "SELECT '\"'||rolname||'\" \"'||rolpassword||'\"' FROM pg_authid WHERE rolname='second_brain';" \ > deploy/pgbouncer/userlist.txt @@ -46,12 +55,14 @@ docker compose -f deploy/docker-compose.prod.yml exec db \ ## 5. Bring up the whole stack ```bash -docker compose -f deploy/docker-compose.prod.yml --env-file deploy/.env.prod up -d --build +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" + +$DC up -d --build ``` The `api` service applies migrations (`alembic upgrade head`, against the DB directly — not via PgBouncer) then starts uvicorn. Services: db, pgbouncer (6432), redis, api (8000), frontend -(3000), prometheus (9090), grafana (3001). Put a reverse proxy (Caddy/Traefik) + TLS in front; -only expose 80/443 publicly, keep 9090/3001 behind the proxy or an SSH tunnel. +(3000), prometheus (9090), grafana (3001), caddy (80/443). Caddy is the public HTTPS entrypoint; +the VPS override keeps the direct app and monitoring ports bound to localhost. ## 6. Verify ```bash @@ -67,7 +78,7 @@ The `worker` service drains the jobs queue continuously; a host cron line enqueu ```cron # /etc/cron.d/second-brain-briefing — 07:00 server time, daily # /etc/cron.d format requires a user field (here: root) between the schedule and the command. -0 7 * * * root cd /path/to/second-brain && docker compose -f deploy/docker-compose.prod.yml --env-file deploy/.env.prod exec -T worker python -m app.jobs.enqueue briefing >> /var/log/second-brain-briefing.log 2>&1 +0 7 * * * root cd /root/second-brain && docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod exec -T worker python -m app.jobs.enqueue briefing >> /var/log/second-brain-briefing.log 2>&1 ``` Read it next morning at `GET /briefing` (or the frontend page). Each run summarizes documents ingested since the previous briefing's `period_end`; a re-run over an empty tail is a cheap @@ -78,11 +89,14 @@ queue any time: `SELECT id,type,status,attempts,last_error FROM jobs ORDER BY id ## 8. Rollback ```bash git checkout -docker compose -f deploy/docker-compose.prod.yml --env-file deploy/.env.prod up -d --build +DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" + +$DC up -d --build # DB: only if a migration must be undone -> alembic downgrade -1 (see backup-restore first) # Prompt rollback needs no deploy: set SECOND_BRAIN_PROMPT_VERSION=rag-v1 and restart api (ADR-0009) ``` ## Update flow (steady state) -`git pull` a green commit → `up -d --build` → verify `/health` + Grafana. Take a DB backup -before any release that includes a migration (see `backup-restore.md`). +`git pull` a green commit → run the canonical `$DC up -d --build` command above → verify +`/api/health` + Grafana. Take a DB backup before any release that includes a migration (see +`backup-restore.md`). diff --git a/docs/runbooks/incident-response.md b/docs/runbooks/incident-response.md index 5809e05..eb29f83 100644 --- a/docs/runbooks/incident-response.md +++ b/docs/runbooks/incident-response.md @@ -5,20 +5,21 @@ Grafana "Second Brain — service overview" dashboard and the Prometheus alerts. ## Triage order 1. `curl -s localhost:8000/health` → is `db` `ok`? -2. `docker compose -f deploy/docker-compose.prod.yml ps` → which containers are up/healthy? -3. `docker compose ... logs --tail=200 ` → recent errors. -4. Grafana dashboard → request rate, p95 latency, 5xx ratio, API up. +2. `DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod"` → use this for all stack commands. +3. `$DC ps` → which containers are up/healthy? +4. `$DC logs --tail=200 ` → recent errors. +5. Grafana dashboard → request rate, p95 latency, 5xx ratio, API up. ## Alert → likely cause → action ### `ApiDown` (Prometheus can't scrape the API) -- `docker compose ... ps api` / `logs api`. Common: DB not healthy yet, or a bad migration on +- `$DC ps api` / `$DC logs api`. Common: DB not healthy yet, or a bad migration on the startup `alembic upgrade head` step. - DB down → see "DB unreachable". Migration failure → restore pre-migrate dump (`backup-restore.md`) or `alembic downgrade -1`, then redeploy a green commit. ### `HighErrorRate` (5xx > 5%) -- `logs api` for tracebacks. If it started after a deploy → roll back to the previous green SHA +- `$DC logs api` for tracebacks. If it started after a deploy → roll back to the previous green SHA (`deploy-checklist.md` §7). - If Gemini-related (quota/timeout): flip to private mode `SECOND_BRAIN_LLM_PROVIDER=ollama` (if Ollama is present) or wait out the quota; chat refuses gracefully on no-context. @@ -30,7 +31,7 @@ Grafana "Second Brain — service overview" dashboard and the Prometheus alerts. index after a schema change → `EXPLAIN ANALYZE` the slow query. ### DB unreachable -- `docker compose ... logs db`. Disk full is the classic cause: `df -h`; prune old WAL/backups, +- `$DC logs db`. Disk full is the classic cause: `df -h`; prune old WAL/backups, Docker images (`docker system prune`). On 4 GB boxes, OOM can kill Postgres — check `dmesg`, add swap, lower Prometheus retention. diff --git a/docs/screenshots/ui-chat-answer.png b/docs/screenshots/ui-chat-answer.png new file mode 100644 index 0000000..8e65391 Binary files /dev/null and b/docs/screenshots/ui-chat-answer.png differ diff --git a/docs/screenshots/ui-chat.png b/docs/screenshots/ui-chat.png new file mode 100644 index 0000000..042c475 Binary files /dev/null and b/docs/screenshots/ui-chat.png differ diff --git a/docs/screenshots/ui-home.png b/docs/screenshots/ui-home.png new file mode 100644 index 0000000..042c475 Binary files /dev/null and b/docs/screenshots/ui-home.png differ