diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4617038..19a5bde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,29 @@ env: PG_DSN: postgresql+psycopg://second_brain:second_brain@localhost:5432/second_brain jobs: + compose-config: + name: Compose config render + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Render production + VPS template + env: + POSTGRES_USER: second_brain + POSTGRES_PASSWORD: dummy_postgres_password + POSTGRES_DB: second_brain + SECOND_BRAIN_API_TOKEN: dummy_api_token + SECOND_BRAIN_ADMIN_TOKEN: dummy_admin_token + SECOND_BRAIN_LLM_PROVIDER: fake + SECOND_BRAIN_GEMINI_API_KEY: dummy_gemini_key + NEXT_PUBLIC_API_BASE_URL: https://127.0.0.1.sslip.io/api + SECOND_BRAIN_CORS_ORIGINS: '["https://127.0.0.1.sslip.io"]' + CADDY_SITE_ADDRESS: 127.0.0.1.sslip.io + run: | + docker compose -p second-brain \ + -f deploy/docker-compose.prod.yml \ + -f deploy/docker-compose.vps.yml.example \ + config >/tmp/second-brain-compose.yml + unit-tests: name: Unit tests (DB-free) runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index a987868..1ec8674 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ mlflow.db # OS Thumbs.db .DS_Store +*.log # Local agent/tooling dirs (not project artifacts) .claude/ diff --git a/AGENTS.md b/AGENTS.md index 2ce0902..4ac014b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,8 +22,11 @@ integrated. JSONB + materialized-view analytics + RLS/audit. Redis for caching/rate-limit only. - **Backend:** Python + FastAPI. **Frontend:** Next.js + TypeScript. **Agent tools:** MCP server. - **MLOps:** MLflow for eval + prompt/model versioning. **CI/CD:** GitHub Actions, eval-gated. -- **Observability:** self-hosted Prometheus + Grafana. -- **Runtime:** ONE small VPS (~$4–6/mo), everything in Docker Compose. Keep cost minimal. +- **Observability:** Prometheus-compatible metrics plus retained Prometheus/Grafana configs; run + dashboards locally or during optional demos instead of paying for always-on monitoring. +- **Runtime:** local-first Docker Compose. Run the stack on demand on the owner's machine; any + VPS/cloud deployment is optional, temporary, and must be explicitly approved before it creates + a recurring bill. - **Kubernetes** is a LEARNING TRACK only (Phase 7): real manifests + HPA + ingress + CI/CD proven on free local k3s/kind, then torn down. NOT the production runtime. Managed-cluster (GKE/EKS) demo is optional and must be deleted immediately after. @@ -36,15 +39,15 @@ integrated. 3. Evaluation + MLOps: eval set, MLflow harness, A/B, prompt versioning + rollback 4. MCP server + agentic actions incl. self-research tool 5. Daily briefing + scheduled pipelines -6. Productionize on VPS + data-ops hardening (RLS, retention, pooling, query tuning) +6. Operations hardening + optional cloud deploy recipe (RLS, retention, pooling, query tuning) 7. Kubernetes learning track on local k3s/kind ## How to work - **Engineer-grade:** write ADRs for real decisions, tests alongside code, keep a deploy checklist. Use established engineering workflows where they fit. -- **Cost-conscious:** never propose anything with a recurring bill beyond the one VPS - without flagging it explicitly and waiting for my OK. +- **Cost-conscious:** never propose anything with a recurring infrastructure bill without + flagging it explicitly and waiting for my OK. - **Incremental:** end each working chunk with something runnable or reviewable, and tell me how to run/verify it. Don't dump huge unrunnable scaffolds. - **Ask before assuming** on anything that affects architecture, cost, or data privacy. @@ -52,7 +55,7 @@ integrated. dollar figure shown in tooling is an *estimate of equivalent API cost*, not a billed amount. Do NOT pause, check in, or ask for re-approval based on session/token cost (including when it "doubles" or crosses a threshold). The "cost-conscious" rule above - refers ONLY to recurring infrastructure bills (e.g. the VPS) — flag those, not session cost. + refers ONLY to recurring infrastructure bills — flag those, not session cost. - **Keep records current:** update `docs/PROGRESS.md` at the end of each session (status + dated log), and append to `docs/implementation-notes.md` whenever you make a decision, change, or trade-off that wasn't in the spec (what / why / what I gave up). @@ -64,7 +67,7 @@ integrated. tier, which is separate from my subscription. - I may use multiple coding agents (Claude, Codex, Antigravity) on this repo — this file is the shared source of truth; `CLAUDE.md` just points here. -- [TODO: add your OS, strongest languages, and chosen VPS once decided.] +- [TODO: add your OS and strongest languages.] ## Note for non-Claude agents diff --git a/README.md b/README.md index c12ef88..565bb25 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,20 @@ # Second Brain -**A personal, always-on AI assistant for streaming cited RAG, hybrid search, daily briefings, and MCP-powered actions.** +**A local-first personal AI assistant for streaming cited RAG, hybrid search, briefings, and MCP-powered actions.** Second Brain captures web passages and personal knowledge, stores them in PostgreSQL with pgvector -and full-text indexes, serves citation-validated answers over SSE, produces a morning briefing, and exposes -agentic tools over MCP. The live deployment runs on one VPS behind Caddy HTTPS with single-owner -API-token authentication, a separate admin-token data-ops guard, and documented backup, restore, -rollback, firewall, and secret-rotation operations. +and full-text indexes, serves citation-validated answers over SSE, produces briefings, and exposes +agentic tools over MCP. The default runtime is local-first Docker Compose so the owner can run it +on demand without a recurring server bill; the old VPS/Caddy deployment recipe is retained only as +an optional cloud demo path. -[![Status](https://img.shields.io/badge/status-live-brightgreen)](docs/USAGE.md) +[![Status](https://img.shields.io/badge/status-local--first-brightgreen)](docs/USAGE.md) [![Roadmap](https://img.shields.io/badge/roadmap-7%2F7%20complete-success)](docs/PROGRESS.md) [![Stack](https://img.shields.io/badge/stack-FastAPI%20%7C%20Postgres%2Bpgvector%20%7C%20Next.js-blue)](#tech-stack) -[![TLS](https://img.shields.io/badge/TLS-Caddy%20auto--HTTPS-0F9D58)](deploy/caddy/Caddyfile) +[![Cloud Demo](https://img.shields.io/badge/cloud%20demo-optional-lightgrey)](deploy/caddy/Caddyfile) [![CI](https://img.shields.io/badge/CI-unit%20%2B%20integration%20%2B%20eval--gated-blue)](.github/workflows) -[![Runtime](https://img.shields.io/badge/runtime-single%20VPS%20%2B%20Docker%20Compose-success)](#production-architecture) +[![Runtime](https://img.shields.io/badge/runtime-local--first%20Docker%20Compose-success)](#runtime-architecture) @@ -27,25 +27,24 @@ rollback, firewall, and secret-rotation operations. Chat over personal notes with inline source citations. -> **Live deployment:** verified on a 2 GB DigitalOcean droplet with Caddy, real Let's Encrypt -> HTTPS via `sslip.io`, hosted Gemini embeddings, and the full 9-service stack running end to end. -> See [docs/USAGE.md](docs/USAGE.md) for live URLs, health checks, backups, rollback, and -> production operations. +> **Runtime decision:** Second Brain now defaults to local/on-demand operation to avoid paying for +> idle cloud uptime. A 2 GB DigitalOcean + Caddy deployment was previously verified and remains as +> an optional recipe, but it is no longer the recommended default. ## Current Status -Last README synchronization: **2026-06-05**. Live deployment last verified: -**2026-06-02**. +Last README synchronization: **2026-06-05**. Runtime default changed to local-first on +**2026-06-05**. | Area | Status | Notes | |---|---:|---| | Product roadmap | Complete | Phases 0-7 are implemented and documented in [docs/PROGRESS.md](docs/PROGRESS.md). | -| Production deployment | Live | Docker Compose on one VPS, fronted by Caddy HTTPS, with localhost-only direct service ports. | -| Production operations | Documented | Bearer-token API access, `ufw` 22/80/443 allow-list, automated DB backup cron, restore drill, health checks, secret rotation, and rollback runbooks. | -| Web UI | Live | Streaming chat, capture, search, ingest, briefing, tasks, research, sources, feedback review, and admin data-ops pages. | -| API | Live | Capture, ingest, streaming and non-streaming chat, search, conversations, feedback analytics, briefing, tasks, research jobs, sources, health, and governed data-ops endpoints. | -| MCP server | Live | `search_notes`, `list_tasks`, and `send_digest` are available by default; `create_task` and `research_topic` require explicit local mutation opt-in. | -| Background jobs | Live | Durable Postgres job queue for daily briefing and async research. | +| Runtime | Local-first | Run the Compose-backed app on demand locally; optional cloud deploy recipe remains for demos. | +| Operations | Documented | Bearer-token API access, backup/restore, health checks, secret rotation, rollback, and optional VPS hardening runbooks. | +| Web UI | Implemented | Streaming chat, capture, search, ingest, briefing, tasks, research, sources, feedback review, and admin data-ops pages. | +| API | Implemented | Capture, ingest, streaming and non-streaming chat, search, conversations, feedback analytics, briefing, tasks, research jobs, sources, health, and governed data-ops endpoints. | +| MCP server | Implemented | `search_notes`, `list_tasks`, and `send_digest` are available by default; `create_task` and `research_topic` require explicit local mutation opt-in. | +| Background jobs | Implemented | Durable Postgres job queue for briefing and async research; schedule locally when desired. | | CI/CD | Active | Unit tests, integration tests against pgvector, and deterministic eval gate. | | Kubernetes | Complete as learning track | Manifests, ingress, HPA, monitoring, and CI smoke on local kind; not production runtime. | @@ -56,13 +55,16 @@ Most recent first. Full detail lives in [docs/PROGRESS.md](docs/PROGRESS.md) and | Update | Summary | Reference | |---|---|:---:| +| Agentic RAG v1 | Added an opt-in read-only LangGraph retrieval graph that plans subqueries, searches existing notes, returns compact trace metadata, and is eval-comparable against regular RAG. | [ADR-0016](docs/adr/0016-agentic-rag-v1.md) | +| Demo loop tooling | Added a seed command for the capture -> chat -> feedback flow and an exporter that turns durable `eval_cases` rows into reviewable YAML fragments for CI dataset patches. | [case study](docs/case-study.md) | +| Runtime strategy update | Changed the default runtime from always-on VPS to local-first/on-demand Docker Compose to avoid recurring idle cloud cost. VPS docs remain only as an optional demo/deploy recipe. | [ADR-0015](docs/adr/0015-local-first-runtime.md) | | Frictionless capture | Added `/capture` API and UI for saving a URL, title, notes, tags, and selected text into the normal bookmark ingest path so captures are searchable and citeable. | [usage](docs/USAGE.md) | | Single-owner authentication | Personal-data routes require `SECOND_BRAIN_API_TOKEN` in production; destructive data-ops also require `SECOND_BRAIN_ADMIN_TOKEN`. Local dev remains keyless unless a token is set. | [usage](docs/USAGE.md) | | Production operations hardening | Added `ufw` firewall steps, automated backup cron template, restore drill, health checks, secret rotation, and rollback procedures. | [runbooks](docs/runbooks/) | | Streaming chat | Added SSE `/chat/stream`; the backend now buffers provider chunks until citation and support validation pass, then emits safe deltas plus the same final contract as `/chat`. | [usage](docs/USAGE.md) | | Local env hygiene | Clarified local Gemini key entry points and tightened `.env` ignore rules while keeping example templates committed. | [progress](docs/PROGRESS.md) | | App surfaces and Redis paths | Added first-class web pages for operating the app, feedback review workflows, source-backed research, weak-context refusal, and optional Redis-backed rate limits/caches. | [PR #20](https://github.com/tomnguyen103/second-brain/pull/20) | -| Live VPS deployment | Caddy reverse proxy, real HTTPS through `sslip.io`, localhost-only direct service ports, and end-to-end production verification. | [PR #14](https://github.com/tomnguyen103/second-brain/pull/14) | +| Optional VPS deployment | Caddy reverse proxy, real HTTPS through `sslip.io`, localhost-only direct service ports, and end-to-end verification. Kept as an optional recipe, not the default runtime. | [PR #14](https://github.com/tomnguyen103/second-brain/pull/14) | | Kubernetes learning track | Local kind manifests, ingress, HPA, monitoring, and GitHub Actions smoke workflow were completed and torn down. | [PR #11](https://github.com/tomnguyen103/second-brain/pull/11) | ## Product Capabilities @@ -70,6 +72,7 @@ Most recent first. Full detail lives in [docs/PROGRESS.md](docs/PROGRESS.md) and | Capability | What is implemented | |---|---| | Streaming cited RAG chat | `/chat/stream` buffers provider chunks until citation/support validation passes, then sends safe deltas and a persisted completion with `[n]` citation markers. `/chat` remains available as the non-streaming fallback. | +| Agentic RAG | Opt-in `/chat` mode uses LangGraph to plan 2-4 note searches, merge evidence, optionally retry weak evidence, and answer through the same citation validator. It is read-only and disabled by default until eval beats baseline. | | Web capture | `/capture` saves a URL, title, selected text, notes, and tags as a `bookmark` source/document through the ingest pipeline; it performs no server-side scraping. | | Hybrid search | pgvector semantic search and PostgreSQL full-text search are fused with reciprocal rank fusion, with configurable weak-context refusal. | | Source ingestion | `/ingest` accepts text documents, dedupes by content hash, chunks semantically, embeds, tags, and stores them. | @@ -77,12 +80,12 @@ Most recent first. Full detail lives in [docs/PROGRESS.md](docs/PROGRESS.md) and | MCP tools | A stdio MCP server exposes search, task listing, and digest composition by default; durable task/research mutations are opt-in for trusted local clients. | | Self-research | `research_topic` can use pasted source text or safe public text/HTML URLs on default HTTP(S) ports, stores provenance, and indexes the resulting `research_note`. | | Evaluation and MLOps | Fixed eval set, MLflow logging, prompt versioning, A/B configs, rollback by env var, and CI eval gate. | -| Feedback quality review | Feedback analytics and negative-feedback review endpoints turn thumbs into reviewable eval candidates. | +| Feedback quality review | Feedback analytics and negative-feedback review endpoints turn thumbs into reviewable eval candidates; reviewed promotions are stored durably in Postgres and exportable as YAML patch fragments. | | Redis paths | Optional Redis-backed `/chat` and `/ingest` rate limits, `/search` response caching, and embedding caching are enabled in production; rate limits fail closed by default. | -| Data governance | RLS, audit logging, retention purge, source export, and source erasure endpoints. | +| Data governance | RLS, audit logging, raw-text retention purge, source export, source erasure, and durable reviewed eval cases. | | Single-owner auth | `SECOND_BRAIN_API_TOKEN` protects chat, conversations, capture, ingest, search, briefing, feedback, tasks, research, sources, and admin surfaces; destructive data-ops also require `X-Second-Brain-Admin-Token: `. | | Observability | Prometheus-format request, cache, and rate-limit metrics at `/metrics`; alert rules and Grafana dashboard configs are retained under `deploy/`, but production Compose does not start monitoring containers until a scanned-clean runtime is selected. | -| Production operations | Docker Compose stack, Caddy HTTPS, bearer-token API access, `ufw` hardening, automated backup cron, restore drill, secret rotation, rollback, and incident response runbooks. | +| Operations artifacts | Local/on-demand runtime guidance plus optional Docker Compose/Caddy deploy recipe, bearer-token API access, backup/restore, secret rotation, rollback, and incident response runbooks. | ## User Surfaces @@ -92,8 +95,27 @@ Most recent first. Full detail lives in [docs/PROGRESS.md](docs/PROGRESS.md) and | API | `/docs` or `/api/*` in production | Capture, ingest, chat, search, briefing, conversations, feedback analytics, tasks, research jobs, sources, and admin data-ops. Personal-data calls require the API bearer token in production. | | MCP | `python -m app.mcp_server` | Tool interface for MCP clients such as Claude Desktop. | | Worker | `python -m app.jobs.worker --loop` | Runs in production; drains briefing and research jobs. | +| Demo tools | `python -m app.demo.seed`, `python -m app.eval.export_cases` | Seed the portfolio loop, then export promoted `eval_cases` rows into a reviewable YAML fragment. | | Runbooks | [docs/runbooks/](docs/runbooks/) | Deploy, firewall, backup/restore, restore drills, secret rotation, rollback, and incident response procedures. | +## Portfolio Demo Loop + +From `backend/`, seed the tight case-study flow: + +```bash +python -m app.demo.seed +``` + +Then open `/feedback`, promote the seeded negative example after reviewing labels, and export the +staged eval rows: + +```bash +python -m app.eval.export_cases --output eval/promoted-cases.yaml +``` + +That gives you a reviewable patch fragment to copy into `eval/dataset.yaml` when a promoted case +should become part of the CI eval gate. + ## Tech Stack | Layer | Choice | @@ -102,6 +124,7 @@ Most recent first. Full detail lives in [docs/PROGRESS.md](docs/PROGRESS.md) and | Frontend | Next.js, TypeScript, Tailwind CSS, shadcn/ui, TanStack Query | | Database | Self-hosted PostgreSQL with pgvector, full-text search, JSONB, RLS, and audit tables | | Retrieval | Hybrid pgvector cosine search plus PostgreSQL full-text search, fused by RRF | +| Agentic orchestration | LangGraph request-scoped `StateGraph` for opt-in read-only agentic RAG | | LLM generation | `gemini-2.5-flash` by default; local Ollama private mode and fake driver behind the same `LLMClient` interface | | Embeddings | Local MiniLM or hosted `gemini-embedding-001`, both normalized to 384 dimensions | | Agent tooling | MCP server over stdio, with durable mutations disabled unless `SECOND_BRAIN_MCP_ENABLE_MUTATIONS=true` | @@ -109,7 +132,7 @@ Most recent first. Full detail lives in [docs/PROGRESS.md](docs/PROGRESS.md) and | Pooling/cache | SQLAlchemy/Postgres connection pooling; Redis powers optional rate limits plus search and embedding caches | | MLOps | Local MLflow file store, eval harness, prompt registry, A/B configs, CI eval gate | | Observability | Prometheus metrics endpoint plus retained Prometheus/Grafana config artifacts; monitoring containers are not part of the production Compose runtime | -| Production runtime | Docker Compose on one VPS | +| Runtime | Local-first Docker Compose; optional single-box VPS recipe for demos | | Kubernetes | Local kind learning track with manifests, ingress, HPA, and CI smoke test | ## Roadmap @@ -123,48 +146,42 @@ Most recent first. Full detail lives in [docs/PROGRESS.md](docs/PROGRESS.md) and | 3 | Evaluation and MLOps: eval set, MLflow, A/B configs, prompt versioning, rollback | Complete | | 4 | MCP server and agentic actions, including self-research | Complete | | 5 | Daily briefing and scheduled pipelines | Complete | -| 6 | VPS productionization, observability, RLS, retention, pooling, query tuning | Complete | +| 6 | Operations hardening, observability, RLS, retention, pooling, query tuning, optional cloud recipe | Complete | | 7 | Kubernetes learning track on local kind/k3s | Complete | -| Live | Caddy HTTPS production deployment on a 2 GB droplet | Live | +| Optional | Caddy HTTPS deployment recipe previously verified on a 2 GB droplet | Available | + +## Runtime Architecture -## Production Architecture +The default system is one local Docker Compose-backed app. For normal use, start Postgres/pgvector, +the API, worker, and frontend on the owner's machine, then stop them when finished. This preserves +the full RAG/MCP/eval architecture without paying for idle uptime. -The production system is one Docker Compose project named `second-brain`. The VPS stack is -`db`, `redis`, `api`, `worker`, `frontend`, and public `caddy`. The API exposes `/metrics` -privately on the box, and the Prometheus/Grafana configs remain in `deploy/` for future -self-hosted monitoring, but production Compose no longer ships vulnerable vendor monitoring -containers as an opt-in profile. +The optional cloud recipe is one Docker Compose project named `second-brain` with `db`, `redis`, +`api`, `worker`, `frontend`, and public `caddy`. It remains useful for a short portfolio demo or +temporary remote access, but it is no longer the default operating model. ```text -Internet HTTPS +Browser / local client | v -+------------------+ -| Caddy | TLS, Let's Encrypt, sslip.io -| reverse proxy | /api/* -> api, /* -> frontend -+---------+--------+ - | - +-----------------------------+ - | | - v v - +-------------+ +-------------+ - | frontend | | api | - | Next.js | | FastAPI | - +-------------+ +------+------+ - | - +-------------------------+------------------+ - | | | - v v v - +-------------+ +-------------+ - | PostgreSQL | | Redis | - | pgvector | | limits/cache| - +-------------+ +-------------+ - ^ - | - +-------------+ - | worker | - | jobs | - +-------------+ ++-------------------+ +-------------------+ +| frontend | API | FastAPI | +| Next.js +------>+ chat/search/etc. | ++-------------------+ +---------+---------+ + | + +-----------------+----------------+ + | | + v v + +-------------+ +-------------+ + | PostgreSQL | | Redis | + | pgvector | | limits/cache| + +------+------+ +-------------+ + ^ + | + +-------------+ + | worker | + | jobs | + +-------------+ ``` @@ -174,25 +191,25 @@ Internet HTTPS second-brain/ |-- README.md |-- AGENTS.md -|-- docker-compose.yml # local dev Postgres + pgvector on host port 5433 +|-- docker-compose.yml # local Postgres + pgvector on host port 5433 |-- backend/ | |-- app/ # api, chat, retrieval, ingest, llm, embeddings, mcp, jobs, eval -| |-- migrations/ # Alembic migrations 0001-0004 +| |-- migrations/ # Alembic migrations 0001-0005 | `-- tests/ # unit and integration tests |-- frontend/ | |-- app/ # chat, capture, search, ingest, briefing, tasks, research, sources, feedback, admin | |-- components/ | `-- lib/api/ |-- deploy/ -| |-- docker-compose.prod.yml # base production stack -| |-- docker-compose.vps.yml.example # Caddy + production binding template +| |-- docker-compose.prod.yml # optional single-box stack +| |-- docker-compose.vps.yml.example # optional Caddy + cloud binding template | |-- caddy/ -| |-- cron/ # installable VPS cron helper scripts +| |-- cron/ # optional host cron helper scripts | |-- prometheus/ | |-- grafana/ | `-- k8s/ # local Kubernetes learning track `-- docs/ - |-- USAGE.md # live operations guide + |-- USAGE.md # local-first usage guide |-- PROGRESS.md # authoritative project status log |-- project-plan.md |-- implementation-notes.md @@ -239,10 +256,13 @@ curl -s http://localhost:8000/health Backend-specific verification is documented in [backend/README.md](backend/README.md). -## Deploy +## Optional Cloud Deploy + +Local/on-demand is the default. Use the cloud deployment only for a deliberate demo, temporary +remote access, or if you explicitly decide the recurring bill is worth it. -The production deployment uses the base compose file plus a VPS-specific override. Always pass -the project name explicitly so Compose does not create a second project from the `deploy/` +The optional cloud deployment uses the base compose file plus a VPS-specific override. Always +pass the project name explicitly so Compose does not create a second project from the `deploy/` directory name. Production secrets live in gitignored `deploy/.env.prod`. Required auth variables: @@ -253,12 +273,13 @@ Production secrets live in gitignored `deploy/.env.prod`. Required auth variable | `SECOND_BRAIN_ADMIN_TOKEN` | Enables export, source deletion, and retention purge when sent as `X-Second-Brain-Admin-Token` alongside the normal API bearer. | ```bash +cp deploy/docker-compose.vps.yml.example deploy/docker-compose.vps.yml DC="docker compose -p second-brain -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml --env-file deploy/.env.prod" $DC up -d --build $DC ps ``` -The full, verified deployment procedure lives in [docs/USAGE.md](docs/USAGE.md), including: +The optional deployment procedure lives in [docs/USAGE.md](docs/USAGE.md), including: - `sslip.io` HTTPS with Caddy and Let's Encrypt - required environment variables @@ -295,6 +316,7 @@ kind delete cluster --name second-brain ## Architecture and Decisions - [docs/project-plan.md](docs/project-plan.md) - full system design and roadmap +- [docs/case-study.md](docs/case-study.md) - tight demo flows from capture to eval gate - [docs/data-model/er-diagram.md](docs/data-model/er-diagram.md) - relational model - [docs/query-optimization.md](docs/query-optimization.md) - measured Postgres tuning notes - [docs/USAGE.md](docs/USAGE.md) - live operations guide @@ -313,27 +335,27 @@ Selected ADRs: ## Cost and Privacy Notes -Second Brain is designed to run on one small VPS. The current verified deployment uses a -2 GB DigitalOcean droplet with hosted Gemini embeddings so the box does not need the local -Torch embedding model in memory. Lower-cost VPS providers remain compatible with the same -Compose architecture. Recent operations hardening stays within the same footprint: host -firewall rules, cron backups, restore drills, and rollback procedures add no recurring -infrastructure cost. +Second Brain is now designed to run locally/on demand by default. That makes the normal recurring +infrastructure cost **$0**: no idle VPS, no managed database, no paid monitoring service. The +previously verified 2 GB DigitalOcean deployment remains compatible with the same Compose +architecture, but keeping it online is an explicit optional cost. Generation uses the configured Gemini API model by default, and embeddings can be either local MiniLM or hosted Gemini embeddings. When hosted Gemini embeddings are enabled, document text is -sent to Google during ingest as well as during chat generation. For a more private mode, use the -local embedding provider and local Ollama generation path, with the trade-off that the VPS needs -more memory. +sent to Google during ingest; during chat, the user question and retrieved chunks are sent to the +configured generation provider. For a more private mode, use the local embedding provider and +local Ollama generation path, with the trade-off that your local machine needs more memory. +Retention nulls only the original `documents.raw_text` copy; searchable chunk text remains until +source erasure. ## Known Follow-Ups - Replace single-owner bearer tokens with account/session auth only if the app becomes multi-user. - Upgrade self-research beyond user-supplied URLs/text into broader external retrieval with source-backed citations. -- Promote reviewed feedback candidates into the fixed eval set and dashboard quality trends. -- Keep restore-drill evidence current and periodically copy backups off the VPS to a trusted - local machine. +- Replace VPS-specific runbook examples with local-first commands where that improves clarity. +- Keep the seeded demo data and case-study screenshots current as the UI changes. +- Keep restore-drill evidence current and keep local database backups somewhere you trust. --- diff --git a/backend/.env.example b/backend/.env.example index 3e05aad..bc3e2c9 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -14,3 +14,6 @@ SECOND_BRAIN_ADMIN_TOKEN=replace-with-long-random-admin-token # MCP durable mutations are off by default. Enable only for trusted local MCP clients. SECOND_BRAIN_MCP_ENABLE_MUTATIONS=false + +# Opt-in read-only agentic RAG experiment. Keep false unless you are comparing it to baseline. +SECOND_BRAIN_AGENTIC_RAG_ENABLED=false diff --git a/backend/README.md b/backend/README.md index 609fd21..c14c94c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -158,10 +158,10 @@ $env:SECOND_BRAIN_GEMINI_API_KEY = "..."; Remove-Item Env:\SECOND_BRAIN_LLM_PROV ## Phase 6 — productionization + data-ops (run & verify) -Data governance (RLS, audit, retention, GDPR export/delete), Prometheus metrics, an eval-gated -CI pipeline, and the prod Compose stack. Services in `app/dataops/*` + `app/obs/*`; admin API in -`app/api/dataops.py`; the prod stack + monitoring config in `deploy/`. See ADR-0011 (VPS) and -ADR-0012 (productionization + governance). +Data governance (RLS, audit, retention, GDPR export/delete, durable reviewed eval cases), +Prometheus metrics, an eval-gated CI pipeline, and the prod Compose stack. Services in +`app/dataops/*` + `app/obs/*`; admin API in `app/api/dataops.py`; the prod stack + monitoring +config in `deploy/`. See ADR-0011 (VPS) and ADR-0012 (productionization + governance). ```powershell cd backend; .\.venv\Scripts\Activate.ps1 @@ -185,6 +185,7 @@ $env:SECOND_BRAIN_ADMIN_TOKEN = "a-long-random-token" # X-Second-Brain-Admin-Token: # DELETE /data/sources/ (GDPR erasure) # POST /admin/retention/purge?older_than_days=180 (null old raw_text) +# POST /feedback/eval-candidates//promote (store reviewed eval case) # 3) Eval gate (the CI quality gate; exit 0 = quality OK) python -m app.eval.gate diff --git a/backend/app/agentic_rag/__init__.py b/backend/app/agentic_rag/__init__.py new file mode 100644 index 0000000..386d28a --- /dev/null +++ b/backend/app/agentic_rag/__init__.py @@ -0,0 +1 @@ +"""Opt-in read-only agentic RAG orchestration (ADR-0016).""" diff --git a/backend/app/agentic_rag/service.py b/backend/app/agentic_rag/service.py new file mode 100644 index 0000000..7e919ab --- /dev/null +++ b/backend/app/agentic_rag/service.py @@ -0,0 +1,582 @@ +"""LangGraph-backed read-only agentic RAG orchestration. + +V1 stays deliberately bounded: plan subqueries, search existing notes, optionally retry the +original question when evidence is weak, then answer through the same citation validator used by +regular chat. It does not call mutation tools, fetch the web, or persist graph checkpoints. +""" +from __future__ import annotations + +import json +import re +import time +from dataclasses import dataclass +from typing import Literal, TypedDict + +from langgraph.graph import END, START, StateGraph +from sqlalchemy.orm import Session + +from app.chat.prompt import ContextItem, build_messages, get_prompt +from app.chat.service import ( + ChatResult, + _PreparedChat, + _finalize_chat, + _history, + _repair_citations_if_needed, +) +from app.config import Settings +from app.db.models import Conversation, Message +from app.llm.base import LLMMessage +from app.retrieval.fusion import FusedHit +from app.retrieval.hybrid import DisplayChunk, hybrid_search, load_display_chunks + +_BULLET_RE = re.compile(r"^\s*(?:[-*]|\d+[.)])\s*") +_FENCED_JSON_RE = re.compile(r"^\s*```(?:json)?\s*(.*?)\s*```\s*$", re.DOTALL | re.IGNORECASE) + + +@dataclass +class _SubqueryResult: + query: str + hits: list[FusedHit] + meta: dict + + +@dataclass +class AgenticAnswerResult: + answer: str + hits: list[FusedHit] + display: dict[int, DisplayChunk] + n_context: int + latency_ms: int + model: str | None + usage: dict + retrieval: dict + messages: list[LLMMessage] + + +class _AgenticState(TypedDict, total=False): + question: str + history: list[LLMMessage] + subqueries: list[str] + subquery_results: list[_SubqueryResult] + hits: list[FusedHit] + display: dict[int, DisplayChunk] + messages: list[LLMMessage] + meta: dict + answer: str + model: str | None + usage: dict + latency_ms: int + planner_failed: bool + verifier_used: bool + verifier_retry: bool + fallback_used: bool + weak_evidence: bool + + +def _clean_query(text: str, max_chars: int) -> str: + cleaned = " ".join((text or "").strip().split()) + cleaned = _BULLET_RE.sub("", cleaned).strip() + if cleaned.startswith(("'", '"')) and cleaned.endswith(("'", '"')): + cleaned = cleaned[1:-1].strip() + return cleaned[:max_chars].strip() + + +def _dedupe_queries(queries: list[str], *, max_queries: int, max_chars: int) -> list[str]: + out: list[str] = [] + seen: set[str] = set() + for query in queries: + cleaned = _clean_query(query, max_chars) + key = cleaned.lower() + if not cleaned or key in seen: + continue + seen.add(key) + out.append(cleaned) + if len(out) >= max_queries: + break + return out + + +def _queries_from_json(value) -> list[str]: # noqa: ANN001 - defensive JSON parser + if isinstance(value, list): + return [str(item) for item in value if isinstance(item, str)] + if isinstance(value, dict): + queries = value.get("queries") + if isinstance(queries, list): + return [str(item) for item in queries if isinstance(item, str)] + return [] + + +def _planner_json_candidates(text: str) -> list[str]: + candidates = [(text or "").strip()] + fence_match = _FENCED_JSON_RE.match(candidates[0]) + if fence_match: + candidates.append(fence_match.group(1).strip()) + start = candidates[0].find("{") + end = candidates[0].rfind("}") + if 0 <= start < end: + candidates.append(candidates[0][start:end + 1]) + return [candidate for candidate in candidates if candidate] + + +def parse_query_plan(text: str, *, question: str, max_queries: int, max_chars: int + ) -> tuple[list[str], bool]: + """Parse planner output into bounded search queries. + + Returns `(queries, failed)`. When parsing fails or produces fewer than two queries, the + original question is included as a conservative fallback query. + """ + raw_queries: list[str] = [] + failed = False + for candidate in _planner_json_candidates(text): + try: + raw_queries = _queries_from_json(json.loads(candidate)) + except json.JSONDecodeError: + continue + if raw_queries: + break + else: + raw_queries = [ + line for line in (text or "").splitlines() + if _clean_query(line, max_chars) + ] + failed = True + + queries = _dedupe_queries(raw_queries, max_queries=max_queries, max_chars=max_chars) + if len(queries) < 2: + queries = _dedupe_queries([*queries, question], max_queries=max_queries, + max_chars=max_chars) + failed = True + return queries or [_clean_query(question, max_chars)], failed + + +def _history_text(history: list[LLMMessage], max_chars: int = 1600) -> str: + lines = [f"{m.role}: {m.content}" for m in history[-6:]] + return "\n".join(lines)[-max_chars:] or "(none)" + + +def _planner_messages(question: str, history: list[LLMMessage], max_queries: int + ) -> list[LLMMessage]: + return [ + LLMMessage( + "system", + "Generate focused search queries for a personal-notes RAG system. " + "Return only JSON in the form {\"queries\":[\"...\"]}. " + f"Produce 2 to {max_queries} concise queries. Do not answer the question.", + ), + LLMMessage( + "user", + "Conversation history:\n" + f"{_history_text(history)}\n\n" + f"Current question:\n{question}", + ), + ] + + +def _verifier_messages(question: str, subqueries: list[str]) -> list[LLMMessage]: + return [ + LLMMessage( + "system", + "You are checking a read-only RAG retrieval plan. If the generated subqueries found " + "no usable evidence, decide whether retrying the user's original wording is useful. " + "Return exactly RETRY or REFUSE.", + ), + LLMMessage( + "user", + "Question:\n" + f"{question}\n\n" + "Subqueries already tried:\n" + + "\n".join(f"- {q}" for q in subqueries), + ), + ] + + +def _merge_method(methods: set[str]) -> str: + if "hybrid" in methods or {"vector", "fulltext"}.issubset(methods): + return "hybrid" + if "fulltext" in methods: + return "fulltext" + return "vector" + + +def _merge_hits(results: list[_SubqueryResult], top_k: int) -> list[FusedHit]: + buckets: dict[int, dict] = {} + for result in results: + seen_in_query: set[int] = set() + for hit in result.hits: + bucket = buckets.setdefault(hit.chunk_id, { + "score": 0.0, + "support": 0, + "methods": set(), + "vector_score": None, + "fulltext_score": None, + }) + bucket["score"] += hit.score + if hit.chunk_id not in seen_in_query: + bucket["support"] += 1 + seen_in_query.add(hit.chunk_id) + bucket["methods"].add(hit.method) + if hit.vector_score is not None: + current = bucket["vector_score"] + bucket["vector_score"] = hit.vector_score if current is None else max( + current, hit.vector_score) + if hit.fulltext_score is not None: + current = bucket["fulltext_score"] + bucket["fulltext_score"] = hit.fulltext_score if current is None else max( + current, hit.fulltext_score) + + merged: list[FusedHit] = [] + for chunk_id, bucket in buckets.items(): + support = int(bucket["support"]) + score = float(bucket["score"]) * (1.0 + 0.10 * max(0, support - 1)) + merged.append(FusedHit( + chunk_id=chunk_id, + score=score, + method=_merge_method(bucket["methods"]), + vector_score=bucket["vector_score"], + fulltext_score=bucket["fulltext_score"], + )) + + merged.sort(key=lambda h: (h.score, h.chunk_id), reverse=True) + merged = merged[:top_k] + for rank, hit in enumerate(merged, start=1): + hit.rank = rank + return merged + + +def _meta_from_results(results: list[_SubqueryResult], hits: list[FusedHit], + *, weak_evidence: bool) -> dict: + candidates_vector = sum(int(r.meta.get("candidates_vector", 0)) for r in results) + candidates_vector_raw = sum(int(r.meta.get("candidates_vector_raw", 0)) for r in results) + candidates_fulltext = sum(int(r.meta.get("candidates_fulltext", 0)) for r in results) + return { + "method": "agentic_hybrid", + "candidates_vector": candidates_vector, + "candidates_vector_raw": candidates_vector_raw, + "candidates_fulltext": candidates_fulltext, + "fused_returned": len(hits), + "weak_context": weak_evidence, + } + + +def _agentic_trace(state: _AgenticState, settings: Settings) -> dict: + results = state.get("subquery_results", []) + return { + "enabled": True, + "strategy": "plan_subsearch_v1", + "subqueries": state.get("subqueries", []), + "subquery_hit_counts": [len(r.hits) for r in results], + "deduped_chunks": len({h.chunk_id for r in results for h in r.hits}), + "selected_chunks": len(state.get("hits", [])), + "weak_evidence": bool(state.get("weak_evidence", False)), + "planner_failed": bool(state.get("planner_failed", False)), + "verifier_used": bool(state.get("verifier_used", False)), + "fallback_used": bool(state.get("fallback_used", False)), + "step_budget": { + "max_subqueries": min(max(1, settings.agentic_rag_max_subqueries), 4), + "recursion_limit": settings.agentic_rag_recursion_limit, + }, + } + + +class _AgenticRagGraph: + def __init__( + self, + db: Session, + embedder, + llm, + settings: Settings, + *, + top_k: int | None, + filters: dict | None, + redis_client, + ) -> None: + self.db = db + self.embedder = embedder + self.llm = llm + self.settings = settings + self.top_k = top_k or settings.retrieval_top_k + self.filters = filters or {} + self.redis_client = redis_client + self.max_queries = min(max(1, settings.agentic_rag_max_subqueries), 4) + self.max_query_chars = settings.retrieval_query_rewrite_max_chars + self.graph = self._build_graph() + + def _search(self, query: str) -> _SubqueryResult: + hits, meta = hybrid_search( + self.db, + self.embedder, + self.settings, + query, + top_k=self.top_k, + source_ids=self.filters.get("source_ids"), + tags=self.filters.get("tags"), + redis_client=self.redis_client, + ) + return _SubqueryResult(query=query, hits=hits, meta=meta) + + def plan_queries(self, state: _AgenticState) -> dict: + try: + response = self.llm.generate( + _planner_messages(state["question"], state.get("history", []), self.max_queries) + ) + queries, failed = parse_query_plan( + response.text, + question=state["question"], + max_queries=self.max_queries, + max_chars=self.max_query_chars, + ) + except Exception: # pragma: no cover - provider/network specific + queries = [_clean_query(state["question"], self.max_query_chars)] + failed = True + return {"subqueries": queries, "planner_failed": failed} + + def retrieve_subqueries(self, state: _AgenticState) -> dict: + return {"subquery_results": [self._search(q) for q in state.get("subqueries", [])]} + + def fallback_retrieve(self, state: _AgenticState) -> dict: + original = _clean_query(state["question"], self.max_query_chars) + tried = {q.lower() for q in state.get("subqueries", [])} + results = list(state.get("subquery_results", [])) + subqueries = list(state.get("subqueries", [])) + if original.lower() not in tried: + subqueries.append(original) + results.append(self._search(original)) + return { + "subqueries": subqueries, + "subquery_results": results, + "fallback_used": True, + } + + def select_context(self, state: _AgenticState) -> dict: + results = state.get("subquery_results", []) + hits = _merge_hits(results, self.top_k) + weak = len(hits) == 0 + return { + "hits": hits, + "weak_evidence": weak, + "meta": _meta_from_results(results, hits, weak_evidence=weak), + } + + def verify_evidence(self, state: _AgenticState) -> dict: + retry = True + try: + response = self.llm.generate( + _verifier_messages(state["question"], state.get("subqueries", [])) + ) + text = (response.text or "").strip().upper() + retry = text != "REFUSE" + except Exception: # pragma: no cover - provider/network specific + retry = True + return {"verifier_used": True, "verifier_retry": retry} + + def answer(self, state: _AgenticState) -> dict: + hits = state.get("hits", []) + display = load_display_chunks(self.db, [h.chunk_id for h in hits]) + items = [ + ContextItem(i + 1, display[h.chunk_id].source_name, + display[h.chunk_id].document_title, display[h.chunk_id].content) + for i, h in enumerate(hits) + ] + messages = build_messages( + state["question"], + items, + state.get("history", []), + prompt_version=self.settings.prompt_version, + ) + started = time.perf_counter() + response = self.llm.generate(messages) + latency_ms = int((time.perf_counter() - started) * 1000) + usage = { + "prompt_tokens": response.prompt_tokens, + "completion_tokens": response.completion_tokens, + "total_tokens": response.total_tokens, + } + return { + "display": display, + "messages": messages, + "answer": response.text, + "model": response.model, + "usage": usage, + "latency_ms": latency_ms, + } + + def refuse(self, state: _AgenticState) -> dict: + return {"weak_evidence": True} + + def _route_after_select(self, state: _AgenticState) -> Literal["answer", "verify", "refuse"]: + if state.get("hits"): + return "answer" + if self.settings.agentic_rag_verifier_enabled and not state.get("fallback_used"): + return "verify" + return "refuse" + + def _route_after_verify(self, state: _AgenticState) -> Literal["retry", "refuse"]: + return "retry" if state.get("verifier_retry", False) else "refuse" + + def _build_graph(self): + builder = StateGraph(_AgenticState) + builder.add_node("plan_queries", self.plan_queries) + builder.add_node("retrieve_subqueries", self.retrieve_subqueries) + builder.add_node("select_context", self.select_context) + builder.add_node("verify_evidence", self.verify_evidence) + builder.add_node("fallback_retrieve", self.fallback_retrieve) + builder.add_node("answer", self.answer) + builder.add_node("refuse", self.refuse) + builder.add_edge(START, "plan_queries") + builder.add_edge("plan_queries", "retrieve_subqueries") + builder.add_edge("retrieve_subqueries", "select_context") + builder.add_conditional_edges("select_context", self._route_after_select, { + "answer": "answer", + "verify": "verify_evidence", + "refuse": "refuse", + }) + builder.add_conditional_edges("verify_evidence", self._route_after_verify, { + "retry": "fallback_retrieve", + "refuse": "refuse", + }) + builder.add_edge("fallback_retrieve", "select_context") + builder.add_edge("answer", END) + builder.add_edge("refuse", END) + return builder.compile() + + def invoke(self, question: str, history: list[LLMMessage]) -> _AgenticState: + initial: _AgenticState = {"question": question, "history": history} + return self.graph.invoke( + initial, + config={"recursion_limit": self.settings.agentic_rag_recursion_limit}, + ) + + +def _persist_agentic_refusal(db: Session, conversation_id: int, settings: Settings, + meta: dict) -> ChatResult: + refusal = get_prompt(settings.prompt_version).refusal_text + assistant = Message(conversation_id=conversation_id, role="assistant", + content=refusal, model=None) + db.add(assistant) + db.commit() + return ChatResult( + conversation_id, + assistant.id, + refusal, + [], + {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + None, + 0, + {**meta, "fused_returned": 0, "refusal_reason": "weak_context"}, + ) + + +def answer_agentic_question( + db: Session, + embedder, + llm, + settings: Settings, + question: str, + *, + history: list[LLMMessage] | None = None, + top_k: int | None = None, + filters: dict | None = None, + redis_client=None, +) -> AgenticAnswerResult: + """Run the agentic graph without writing chat history or retrieval rows.""" + graph = _AgenticRagGraph( + db, + embedder, + llm, + settings, + top_k=top_k, + filters=filters, + redis_client=redis_client, + ) + state = graph.invoke(question, history or []) + trace = _agentic_trace(state, settings) + meta = {**state.get("meta", {}), "agentic": trace} + hits = state.get("hits", []) + if not hits or "answer" not in state: + return AgenticAnswerResult( + answer=get_prompt(settings.prompt_version).refusal_text, + hits=[], + display={}, + n_context=0, + latency_ms=0, + model=None, + usage={"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + retrieval={**meta, "fused_returned": 0, "refusal_reason": "weak_context"}, + messages=[], + ) + return AgenticAnswerResult( + answer=state["answer"], + hits=hits, + display=state["display"], + n_context=len(hits), + latency_ms=state.get("latency_ms", 0), + model=state.get("model"), + usage=state.get("usage", {}), + retrieval=meta, + messages=state.get("messages", []), + ) + + +def agentic_chat( + db: Session, + embedder, + llm, + settings: Settings, + *, + message: str, + conversation_id: int | None = None, + top_k: int | None = None, + filters: dict | None = None, + include_chunks: bool = True, + redis_client=None, +) -> ChatResult: + """Run a bounded read-only agentic RAG turn and persist the final assistant message.""" + if conversation_id is None: + conversation = Conversation(title=message[:80]) + db.add(conversation) + db.flush() + conversation_id = conversation.id + + history = _history(db, conversation_id, settings.history_window) + db.add(Message(conversation_id=conversation_id, role="user", content=message)) + db.flush() + + result = answer_agentic_question( + db, + embedder, + llm, + settings, + message, + history=history, + top_k=top_k, + filters=filters, + redis_client=redis_client, + ) + if not result.hits: + return _persist_agentic_refusal(db, conversation_id, settings, result.retrieval) + + prepared = _PreparedChat( + conversation_id=conversation_id, + messages=result.messages, + hits=result.hits, + display=result.display, + meta=result.retrieval, + item_count=len(result.hits), + include_chunks=include_chunks, + ) + answer, model, usage, latency_ms = _repair_citations_if_needed( + llm, + prepared, + answer=result.answer, + model=result.model, + usage=result.usage, + latency_ms=result.latency_ms, + ) + return _finalize_chat( + db, + prepared, + answer=answer, + model=model, + usage=usage, + latency_ms=latency_ms, + ) diff --git a/backend/app/api/chat.py b/backend/app/api/chat.py index 2c29941..e591280 100644 --- a/backend/app/api/chat.py +++ b/backend/app/api/chat.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import Session from app import deps +from app.agentic_rag.service import agentic_chat from app.cache.rate_limit import check_rate_limit, client_identity from app.chat.service import ChatResult, chat, stream_chat from app.config import Settings @@ -119,18 +120,36 @@ def chat_endpoint( ): _check_chat_rate_limit(request, redis_client, settings) - llm = deps.get_llm_client(settings, private_mode=req.options.private_mode) filters = _filters_from_request(req) - result = chat( - db, embedder, llm, settings, - message=req.message, - conversation_id=req.conversation_id, - top_k=req.top_k, - filters=filters, - include_chunks=req.options.include_chunks, - redis_client=redis_client, - ) + if req.options.agentic and not settings.agentic_rag_enabled: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="agentic RAG is disabled", + ) + + llm = deps.get_llm_client(settings, private_mode=req.options.private_mode) + + if req.options.agentic: + result = agentic_chat( + db, embedder, llm, settings, + message=req.message, + conversation_id=req.conversation_id, + top_k=req.top_k, + filters=filters, + include_chunks=req.options.include_chunks, + redis_client=redis_client, + ) + else: + result = chat( + db, embedder, llm, settings, + message=req.message, + conversation_id=req.conversation_id, + top_k=req.top_k, + filters=filters, + include_chunks=req.options.include_chunks, + redis_client=redis_client, + ) return _chat_response(result) @@ -146,6 +165,12 @@ def chat_stream_endpoint( ): _check_chat_rate_limit(request, redis_client, settings) + if req.options.agentic: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail="streaming is unavailable for agentic RAG", + ) + llm = deps.get_llm_client(settings, private_mode=req.options.private_mode) if not supports_streaming(llm): raise HTTPException( diff --git a/backend/app/api/conversations.py b/backend/app/api/conversations.py index 0e97164..8a9fa91 100644 --- a/backend/app/api/conversations.py +++ b/backend/app/api/conversations.py @@ -7,17 +7,19 @@ from fastapi import APIRouter, Depends, HTTPException, Query, status from sqlalchemy import func, select +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, selectinload from app import deps from app.chat.prompt import parse_citations from app.dataops import audit -from app.db.models import Conversation, Feedback, Message +from app.db.models import Conversation, EvalCaseRecord, Feedback, Message from app.eval.dataset import ( CORPUS_DIR as EVAL_CORPUS_DIR, DEFAULT_DATASET as EVAL_DATASET_PATH, EvalCase, - append_eval_case, + load_dataset, + validate_new_eval_case, ) from app.retrieval.hybrid import load_display_chunks from app.schemas.chat import CitationOut @@ -464,8 +466,13 @@ def promote_feedback_eval_candidate( }, } + fixed_case_ids = { + case.id for case in load_dataset(EVAL_DATASET_PATH, corpus_dir=EVAL_CORPUS_DIR) + } + stored_case_ids = set(db.scalars(select(EvalCaseRecord.case_id)).all()) + try: - reviewed = append_eval_case( + reviewed = validate_new_eval_case( EvalCase( id=req.id, question=req.question, @@ -474,7 +481,7 @@ def promote_feedback_eval_candidate( expect_refusal=req.expect_refusal, review=review, ), - path=EVAL_DATASET_PATH, + existing_ids=fixed_case_ids | stored_case_ids, corpus_dir=EVAL_CORPUS_DIR, ) except ValueError as exc: @@ -482,28 +489,53 @@ def promote_feedback_eval_candidate( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc) ) from exc - audit.record( - db, - actor="eval-reviewer", - action="create", - entity_type="eval_case", - entity_id=feedback_id, - detail={ - "op": "promote_eval_case", - "feedback_id": feedback_id, - "case_id": reviewed.id, - "expected_docs": reviewed.expected_docs, - "expected_keywords": reviewed.expected_keywords, - "expect_refusal": reviewed.expect_refusal, - "review": reviewed.review, - }, - enabled=settings.audit_enabled, + stored_case = EvalCaseRecord( + case_id=reviewed.id, + feedback_id=feedback_id, + question=reviewed.question, + expected_docs=reviewed.expected_docs, + expected_keywords=reviewed.expected_keywords, + expect_refusal=reviewed.expect_refusal, + review=reviewed.review, ) - db.commit() + savepoint = db.begin_nested() + try: + db.add(stored_case) + db.flush() + + audit.record( + db, + actor="eval-reviewer", + action="create", + entity_type="eval_case", + entity_id=stored_case.id, + detail={ + "op": "promote_eval_case", + "storage": "postgres", + "eval_case_record_id": stored_case.id, + "feedback_id": feedback_id, + "case_id": reviewed.id, + "expected_docs": reviewed.expected_docs, + "expected_keywords": reviewed.expected_keywords, + "expect_refusal": reviewed.expect_refusal, + "review": reviewed.review, + }, + enabled=settings.audit_enabled, + ) + if savepoint.is_active: + savepoint.commit() + db.commit() + except IntegrityError as exc: + if savepoint.is_active: + savepoint.rollback() + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"eval case id already exists: {reviewed.id}", + ) from exc return PromoteEvalCandidateResponse( promoted_at=promoted_at, - dataset_path="backend/eval/dataset.yaml", + dataset_path="postgres:eval_cases", case=EvalCandidate( id=reviewed.id, question=reviewed.question, @@ -512,8 +544,10 @@ def promote_feedback_eval_candidate( expect_refusal=reviewed.expect_refusal, metadata={ "feedback_id": feedback_id, + "eval_case_record_id": stored_case.id, "needs_review": False, "promoted_from": "feedback", + "storage": "postgres", "review": reviewed.review, }, ), diff --git a/backend/app/chat/prompt.py b/backend/app/chat/prompt.py index 4e0ec87..d96c13c 100644 --- a/backend/app/chat/prompt.py +++ b/backend/app/chat/prompt.py @@ -49,7 +49,7 @@ class PromptSpec: SYSTEM_PROMPT = _RAG_V1.system_prompt REFUSAL_TEXT = _RAG_V1.refusal_text -_MARKER = re.compile(r"\[(\d+)\]") +_MARKER_GROUP = re.compile(r"\[((?:\s*\d+\s*)(?:,\s*\d+\s*)*)\]") def get_prompt(version: str) -> PromptSpec: @@ -91,16 +91,31 @@ def build_messages(question: str, items: list[ContextItem], return msgs -def all_citation_markers(answer: str) -> list[int]: - """Ordered, de-duplicated markers emitted by the model, regardless of validity.""" +def citation_markers_in_text(text: str) -> list[int]: + """Ordered, de-duplicated markers in one text span, including grouped markers. + + Gemini commonly emits compact citations like ``[1, 2]``. Treat those the same as + ``[1] [2]`` so citation validation follows the marker contract humans see. + """ seen: list[int] = [] - for m in _MARKER.findall(answer): - i = int(m) - if i not in seen: - seen.append(i) + for group in _MARKER_GROUP.findall(text or ""): + for marker in re.findall(r"\d+", group): + i = int(marker) + if i not in seen: + seen.append(i) return seen +def strip_citation_markers(text: str) -> str: + """Remove recognized citation marker groups from text.""" + return _MARKER_GROUP.sub("", text or "") + + +def all_citation_markers(answer: str) -> list[int]: + """Ordered, de-duplicated markers emitted by the model, regardless of validity.""" + return citation_markers_in_text(answer) + + def parse_citations(answer: str, n_items: int) -> list[int]: """Ordered, de-duplicated, in-range markers the model actually used.""" return [i for i in all_citation_markers(answer) if 1 <= i <= n_items] diff --git a/backend/app/chat/service.py b/backend/app/chat/service.py index e786fac..90710da 100644 --- a/backend/app/chat/service.py +++ b/backend/app/chat/service.py @@ -9,7 +9,14 @@ from sqlalchemy import select from sqlalchemy.orm import Session -from app.chat.prompt import ContextItem, all_citation_markers, build_messages, get_prompt +from app.chat.prompt import ( + ContextItem, + all_citation_markers, + build_messages, + citation_markers_in_text, + get_prompt, + strip_citation_markers, +) from app.config import Settings from app.db.models import Conversation, Message, Retrieval from app.llm.base import LLMMessage, supports_streaming @@ -62,7 +69,6 @@ class StreamingUnavailable(RuntimeError): "Please try again." ) -_CITATION_MARKER_RE = re.compile(r"\[(\d+)\]") _SEGMENT_SPLIT_RE = re.compile(r"(?<=[.!?])\s+|\n+") _SUPPORT_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9_'-]{1,}") _SUPPORT_STOPWORDS = { @@ -86,6 +92,27 @@ class _PreparedChat: include_chunks: bool +@dataclass +class _CitationValidation: + cited: list[int] + invalid_markers: list[int] + support_failures: list[dict] + + @property + def failed(self) -> bool: + return bool(self.invalid_markers or not self.cited or self.support_failures) + + @property + def reason(self) -> str | None: + if self.invalid_markers: + return "invalid_citations" + if not self.cited: + return "missing_citations" + if self.support_failures: + return "unsupported_claims" + return None + + def _support_tokens(text: str) -> set[str]: return { token.strip("'") @@ -118,11 +145,10 @@ def _citation_support_failures(answer: str, prepared: _PreparedChat) -> list[dic failures: list[dict] = [] for segment in _claim_segments(answer): markers = [ - int(match.group(1)) - for match in _CITATION_MARKER_RE.finditer(segment) - if 1 <= int(match.group(1)) <= prepared.item_count + marker for marker in citation_markers_in_text(segment) + if 1 <= marker <= prepared.item_count ] - claim_text = _CITATION_MARKER_RE.sub("", segment) + claim_text = strip_citation_markers(segment) claim_tokens = _support_tokens(claim_text) if not claim_tokens: continue @@ -145,6 +171,101 @@ def _citation_support_failures(answer: str, prepared: _PreparedChat) -> list[dic return failures +def _validate_citations(answer: str, prepared: _PreparedChat) -> _CitationValidation: + emitted_markers = all_citation_markers(answer) + invalid_markers = [i for i in emitted_markers if i < 1 or i > prepared.item_count] + cited = [i for i in emitted_markers if 1 <= i <= prepared.item_count] + support_failures = ( + [] if invalid_markers or not cited else _citation_support_failures(answer, prepared) + ) + return _CitationValidation(cited, invalid_markers, support_failures) + + +def _repair_citation_messages(prepared: _PreparedChat, draft: str) -> list[LLMMessage]: + return [ + *prepared.messages, + LLMMessage( + "user", + "Your previous draft failed citation validation. Rewrite it using ONLY the " + "numbered context above.\n\n" + "Rules:\n" + "- Every factual sentence must include bracket citations in that same sentence.\n" + "- Do not write uncited headings, preambles, markdown labels, or transitions.\n" + "- Remove claims that are not directly supported by the numbered context.\n" + "- Return only the rewritten answer.\n\n" + f"Draft to repair:\n{draft}", + ), + ] + + +def _merge_usage(base: dict, extra: dict) -> dict: + merged = dict(base or {}) + for key in ("prompt_tokens", "completion_tokens", "total_tokens"): + left = merged.get(key) + right = (extra or {}).get(key) + if isinstance(left, int) and isinstance(right, int): + merged[key] = left + right + elif left is None: + merged[key] = right + return merged + + +def _repair_citations_if_needed( + llm, + prepared: _PreparedChat, + *, + answer: str, + model: str | None, + usage: dict, + latency_ms: int, +) -> tuple[str, str | None, dict, int]: + validation = _validate_citations(answer, prepared) + if not validation.failed: + return answer, model, usage, latency_ms + + prepared.meta = { + **prepared.meta, + "citation_repair_attempted": True, + "citation_repair_original_reason": validation.reason, + } + try: + started = time.perf_counter() + repaired = llm.generate(_repair_citation_messages(prepared, answer)) + repair_latency_ms = int((time.perf_counter() - started) * 1000) + except Exception: # pragma: no cover - provider/network specific + prepared.meta = { + **prepared.meta, + "citation_repair_succeeded": False, + "citation_repair_error": "provider_error", + } + return answer, model, usage, latency_ms + + repaired_usage = { + "prompt_tokens": repaired.prompt_tokens, + "completion_tokens": repaired.completion_tokens, + "total_tokens": repaired.total_tokens, + } + repaired_validation = _validate_citations(repaired.text, prepared) + prepared.meta = { + **prepared.meta, + "citation_repair_succeeded": not repaired_validation.failed, + "citation_repair_latency_ms": repair_latency_ms, + } + if repaired_validation.failed: + prepared.meta = { + **prepared.meta, + "citation_repair_failure_reason": repaired_validation.reason, + } + return answer, model, _merge_usage(usage, repaired_usage), latency_ms + repair_latency_ms + + return ( + repaired.text, + repaired.model or model, + _merge_usage(usage, repaired_usage), + latency_ms + repair_latency_ms, + ) + + def _history(db: Session, conversation_id: int, window: int) -> list[LLMMessage]: rows = db.scalars( select(Message).where(Message.conversation_id == conversation_id) @@ -196,26 +317,16 @@ def _prepare_chat(db: Session, embedder, llm, settings: Settings, *, message: st def _finalize_chat(db: Session, prepared: _PreparedChat, *, answer: str, model: str | None, usage: dict, latency_ms: int) -> ChatResult: - emitted_markers = all_citation_markers(answer) - invalid_markers = [i for i in emitted_markers if i < 1 or i > prepared.item_count] - cited = [i for i in emitted_markers if 1 <= i <= prepared.item_count] - support_failures = ( - [] if invalid_markers or not cited else _citation_support_failures(answer, prepared) - ) - if invalid_markers or not cited or support_failures: + validation = _validate_citations(answer, prepared) + cited = validation.cited + if validation.failed: answer = CITATION_FAILURE_TEXT prepared.meta = { **prepared.meta, "citation_validation_failed": True, - "citation_failure_reason": ( - "invalid_citations" - if invalid_markers - else "missing_citations" - if not cited - else "unsupported_claims" - ), - "invalid_citation_markers": invalid_markers, - "unsupported_citation_segments": support_failures, + "citation_failure_reason": validation.reason, + "invalid_citation_markers": validation.invalid_markers, + "unsupported_citation_segments": validation.support_failures, } cited = [] assistant = Message(conversation_id=prepared.conversation_id, role="assistant", @@ -268,7 +379,15 @@ def chat(db: Session, embedder, llm, settings: Settings, *, message: str, usage = {"prompt_tokens": resp.prompt_tokens, "completion_tokens": resp.completion_tokens, "total_tokens": resp.total_tokens} - return _finalize_chat(db, prepared, answer=resp.text, model=resp.model, + answer, model, usage, latency_ms = _repair_citations_if_needed( + llm, + prepared, + answer=resp.text, + model=resp.model, + usage=usage, + latency_ms=latency_ms, + ) + return _finalize_chat(db, prepared, answer=answer, model=model, usage=usage, latency_ms=latency_ms) @@ -312,13 +431,24 @@ def stream_chat(db: Session, embedder, llm, settings: Settings, *, message: str, delta_parts.append(chunk.text) latency_ms = int((time.perf_counter() - started) * 1000) - raw_answer = "".join(parts) - result = _finalize_chat(db, prepared, answer=raw_answer, model=model, + streamed_answer = "".join(parts) + final_answer, model, usage, latency_ms = _repair_citations_if_needed( + llm, + prepared, + answer=streamed_answer, + model=model, + usage=usage, + latency_ms=latency_ms, + ) + result = _finalize_chat(db, prepared, answer=final_answer, model=model, usage=usage, latency_ms=latency_ms) # Retrieved notes are untrusted input. Do not send provider chunks until the # assembled answer passes citation validation, otherwise prompt-injected text # could leak over SSE before the final response is replaced. - if not result.retrieval.get("citation_validation_failed") and result.answer == raw_answer: + if ( + not result.retrieval.get("citation_validation_failed") + and result.answer == streamed_answer + ): for text in delta_parts: yield ChatStreamEvent(type="delta", text=text) yield ChatStreamEvent(type="complete", result=result) diff --git a/backend/app/config.py b/backend/app/config.py index a788aca..b81a542 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -53,6 +53,11 @@ class Settings(BaseSettings): # Chat (ADR-0006) history_window: int = 6 + # Opt-in read-only agentic RAG (ADR-0016). Disabled by default until eval beats baseline. + agentic_rag_enabled: bool = False + agentic_rag_max_subqueries: int = 4 + agentic_rag_verifier_enabled: bool = True + agentic_rag_recursion_limit: int = 8 # Prompt versioning (ADR-0009) — active prompt; rollback = set this back to a prior version prompt_version: str = "rag-v1" # rag-v1 | rag-v2 @@ -62,7 +67,7 @@ class Settings(BaseSettings): mlflow_experiment: str = "second-brain-rag" # API - cors_origins: list[str] = ["http://localhost:3000"] + cors_origins: list[str] = ["http://localhost:3000", "http://127.0.0.1:3000"] # Single-user API bearer token for notes, conversations, sources, tasks, feedback, and # research endpoints. None keeps local/dev tests keyless; prod compose requires it. api_token: str | None = None diff --git a/backend/app/dataops/retention.py b/backend/app/dataops/retention.py index 09bc410..0def73f 100644 --- a/backend/app/dataops/retention.py +++ b/backend/app/dataops/retention.py @@ -1,9 +1,9 @@ """raw_text retention (ADR-0012, D4). -After a document is embedded, its chunks + embeddings carry the retrievable signal — the -original `raw_text` is the only PII-bearing free text we keep, and per the retention policy it -is nulled `retention_raw_text_days` after ingestion. Chunks and embeddings are NOT removed -(that would break search); retention != erasure (see erasure.py for delete-my-data). +After a document is embedded, chunks + embeddings carry the retrievable signal. This retention +path nulls the original `documents.raw_text` copy after `retention_raw_text_days`; it does not +anonymize the document because `chunks.content` intentionally remains searchable. Retention != +erasure (see erasure.py for source-level delete-my-data). """ from __future__ import annotations diff --git a/backend/app/db/models.py b/backend/app/db/models.py index f6cc781..95fe915 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -10,6 +10,7 @@ from pgvector.sqlalchemy import Vector from sqlalchemy import ( BigInteger, + Boolean, CheckConstraint, Computed, DateTime, @@ -231,6 +232,32 @@ class Feedback(Base): message: Mapped["Message"] = relationship(back_populates="feedback") +class EvalCaseRecord(Base): + """A reviewed feedback-derived eval case persisted durably in Postgres. + + The source-controlled `backend/eval/dataset.yaml` remains the fixed CI gate dataset. + This table is the production-safe review ledger for cases promoted from live feedback. + """ + __tablename__ = "eval_cases" + __table_args__ = ( + UniqueConstraint("case_id", name="uq_eval_cases_case_id"), + Index("ix_eval_cases_feedback_id", "feedback_id"), + ) + id: Mapped[int] = _pk() + case_id: Mapped[str] = mapped_column(Text, nullable=False) + feedback_id: Mapped[int | None] = mapped_column( + BigInteger, ForeignKey("feedback.id", ondelete="SET NULL") + ) + question: Mapped[str] = mapped_column(Text, nullable=False) + expected_docs: Mapped[list[str]] = mapped_column(JSONB, nullable=False, server_default="[]") + expected_keywords: Mapped[list[str]] = mapped_column( + JSONB, nullable=False, server_default="[]" + ) + expect_refusal: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default="false") + review: Mapped[dict] = mapped_column(JSONB, nullable=False, server_default="{}") + created_at: Mapped[datetime] = _created() + + class AuditLog(Base): __tablename__ = "audit_log" __table_args__ = ( diff --git a/backend/app/demo/__init__.py b/backend/app/demo/__init__.py new file mode 100644 index 0000000..adbbacd --- /dev/null +++ b/backend/app/demo/__init__.py @@ -0,0 +1 @@ +"""Demo utilities for reproducible portfolio flows.""" diff --git a/backend/app/demo/seed.py b/backend/app/demo/seed.py new file mode 100644 index 0000000..168bd10 --- /dev/null +++ b/backend/app/demo/seed.py @@ -0,0 +1,135 @@ +"""Seed a compact capture -> chat -> feedback demo flow. + +Run from `backend/`: + + python -m app.demo.seed +""" +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from typing import Sequence + +from sqlalchemy.orm import Session + +from app.cache.redis_client import get_redis_client +from app.capture.service import capture_page +from app.chat.service import chat +from app.config import Settings, settings +from app.db.models import Feedback +from app.schemas.capture import CaptureRequest + + +DEMO_URL = "https://example.com/second-brain-demo" +DEMO_TITLE = "Second Brain Demo: Feedback Eval Loop" +DEMO_SELECTED_TEXT = ( + "Second Brain captures browser-provided passages, retrieves them with hybrid Postgres search, " + "answers with citations, collects thumbs-down feedback, and stages reviewed eval cases before " + "they become source-controlled release gates." +) +DEMO_NOTES = ( + "Use this seeded note to demo the loop: capture, cited chat, feedback review, eval export, " + "and the deterministic eval gate." +) +DEMO_QUESTION = "How does Second Brain turn feedback into eval coverage?" +DEMO_FEEDBACK = "Demo negative feedback: promote this into a reviewed eval case after checking labels." + + +@dataclass +class DemoSeedResult: + source_id: int + document_id: int | None + conversation_id: int + assistant_message_id: int + feedback_id: int + question: str + + +def seed_demo_flow( + db: Session, + embedder, + llm, + cfg: Settings, + *, + redis_client=None, +) -> DemoSeedResult: + capture = capture_page( + db, + embedder, + cfg, + CaptureRequest( + url=DEMO_URL, + title=DEMO_TITLE, + selected_text=DEMO_SELECTED_TEXT, + notes=DEMO_NOTES, + tags=["demo", "eval", "feedback"], + ), + redis_client=redis_client, + ) + if not capture.ingest.documents: + raise RuntimeError("demo capture produced no documents to chat over") + document = capture.ingest.documents[0] + + result = chat( + db, + embedder, + llm, + cfg, + message=DEMO_QUESTION, + top_k=5, + redis_client=redis_client, + ) + + feedback = Feedback(message_id=result.message_id, rating=-1, comment=DEMO_FEEDBACK) + db.add(feedback) + db.commit() + + return DemoSeedResult( + source_id=capture.ingest.source_id, + document_id=document.document_id, + conversation_id=result.conversation_id, + assistant_message_id=result.message_id, + feedback_id=feedback.id, + question=DEMO_QUESTION, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m app.demo.seed", + description="Seed a small capture/chat/feedback demo flow.", + ) + parser.add_argument( + "--real-llm", + action="store_true", + help="use the configured LLM instead of the deterministic fake LLM", + ) + args = parser.parse_args(argv) + + from app.db.session import SessionLocal + from app.deps import get_embedder + from app.llm.factory import get_llm_client + from app.llm.fake import FakeLLMClient + + embedder = get_embedder() + llm = get_llm_client(settings) if args.real_llm else FakeLLMClient() + redis_client = get_redis_client(settings) + + with SessionLocal() as db: + result = seed_demo_flow(db, embedder, llm, settings, redis_client=redis_client) + + print("Seeded Second Brain demo flow") + print(f" source_id: {result.source_id}") + print(f" document_id: {result.document_id}") + print(f" conversation_id: {result.conversation_id}") + print(f" assistant_message_id: {result.assistant_message_id}") + print(f" feedback_id: {result.feedback_id}") + print(f" question: {result.question}") + print("") + print("Next: open /feedback, review the negative case, promote it, then export staged evals:") + print(" python -m app.eval.export_cases --output eval/promoted-cases.yaml") + return 0 + + +if __name__ == "__main__": # pragma: no cover - CLI boundary + raise SystemExit(main()) diff --git a/backend/app/eval/configs.py b/backend/app/eval/configs.py index 82b0383..95b2b35 100644 --- a/backend/app/eval/configs.py +++ b/backend/app/eval/configs.py @@ -17,6 +17,7 @@ class EvalConfig: llm_provider: str # gemini | ollama | fake prompt_version: str # rag-v1 | rag-v2 top_k: int + agentic: bool = False CONFIGS: dict[str, EvalConfig] = { @@ -24,9 +25,11 @@ class EvalConfig: # so any metric delta is attributable to the prompt. Proves the harness + config plumbing. "baseline": EvalConfig("baseline", "fake", "rag-v1", 5), "variant": EvalConfig("variant", "fake", "rag-v2", 5), + "agentic": EvalConfig("agentic", "fake", "rag-v1", 5, True), # Real prompt A/B (needs a Gemini key) — the meaningful quality comparison rag-v1 vs rag-v2. "gemini": EvalConfig("gemini", "gemini", "rag-v1", 5), "gemini-v2": EvalConfig("gemini-v2", "gemini", "rag-v2", 5), + "gemini-agentic": EvalConfig("gemini-agentic", "gemini", "rag-v1", 5, True), } @@ -37,4 +40,5 @@ def settings_for(config: EvalConfig, base: Settings | None = None) -> Settings: "llm_provider": config.llm_provider, "prompt_version": config.prompt_version, "retrieval_top_k": config.top_k, + "agentic_rag_enabled": config.agentic, }) diff --git a/backend/app/eval/dataset.py b/backend/app/eval/dataset.py index fc68fc9..8b1f66c 100644 --- a/backend/app/eval/dataset.py +++ b/backend/app/eval/dataset.py @@ -252,15 +252,31 @@ def append_eval_case( ) -> EvalCase: dataset_path = Path(path) current_text = dataset_path.read_text(encoding="utf-8") - corpus = _corpus_titles(corpus_dir) existing = load_dataset(dataset_path, corpus_dir=corpus_dir) - seen = {item.id for item in existing} - reviewed = _case_from_item(_case_item(case), seen=seen, corpus_titles=corpus) + reviewed = validate_new_eval_case( + case, + existing_ids={item.id for item in existing}, + corpus_dir=corpus_dir, + ) base_text = current_text.rstrip() if base_text.endswith("cases: []"): base_text = base_text[: -len("cases: []")] + "cases:" new_text = base_text + _case_block(reviewed) - _parse_dataset(yaml.safe_load(new_text), corpus_titles=corpus) + _parse_dataset(yaml.safe_load(new_text), corpus_titles=_corpus_titles(corpus_dir)) dataset_path.write_text(new_text, encoding="utf-8") return reviewed + + +def validate_new_eval_case( + case: EvalCase, + *, + existing_ids: set[str] | None = None, + corpus_dir: Path | str | None = CORPUS_DIR, +) -> EvalCase: + """Validate a candidate case against the fixed corpus without writing a dataset file.""" + return _case_from_item( + _case_item(case), + seen=set(existing_ids or set()), + corpus_titles=_corpus_titles(corpus_dir), + ) diff --git a/backend/app/eval/export_cases.py b/backend/app/eval/export_cases.py new file mode 100644 index 0000000..0a0f7fe --- /dev/null +++ b/backend/app/eval/export_cases.py @@ -0,0 +1,129 @@ +"""Export reviewed Postgres eval cases as a source-controlled YAML patch fragment. + +The production promotion endpoint writes `eval_cases` rows. This module gives an operator a +reviewable bridge back to `backend/eval/dataset.yaml` without letting the API mutate repo files. +""" +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Iterable, Sequence + +import yaml +from sqlalchemy import select +from sqlalchemy.orm import Session + +from app.db.models import EvalCaseRecord +from app.eval.dataset import ( + CORPUS_DIR, + DEFAULT_DATASET, + EvalCase, + load_dataset, + validate_new_eval_case, +) + + +def _case_item(case: EvalCase) -> dict: + item = { + "id": case.id, + "question": case.question, + "expected_docs": case.expected_docs, + "expected_keywords": case.expected_keywords, + "expect_refusal": case.expect_refusal, + } + if case.review: + item["review"] = case.review + return item + + +def record_to_eval_case(record: EvalCaseRecord) -> EvalCase: + return EvalCase( + id=record.case_id, + question=record.question, + expected_docs=list(record.expected_docs or []), + expected_keywords=list(record.expected_keywords or []), + expect_refusal=record.expect_refusal, + review=dict(record.review or {}), + ) + + +def load_export_records(db: Session, *, case_ids: Sequence[str] | None = None) -> list[EvalCaseRecord]: + stmt = select(EvalCaseRecord).order_by(EvalCaseRecord.id) + if case_ids: + stmt = stmt.where(EvalCaseRecord.case_id.in_(list(case_ids))) + return list(db.scalars(stmt).all()) + + +def export_cases_fragment( + records: Iterable[EvalCaseRecord], + *, + dataset_path: Path | str = DEFAULT_DATASET, + corpus_dir: Path | str | None = CORPUS_DIR, +) -> str: + """Return a `cases:` YAML fragment for staged rows not already in the fixed dataset.""" + fixed_ids = {case.id for case in load_dataset(dataset_path, corpus_dir=corpus_dir)} + seen = set(fixed_ids) + cases: list[EvalCase] = [] + for record in records: + if record.case_id in fixed_ids: + continue + case = validate_new_eval_case( + record_to_eval_case(record), + existing_ids=seen, + corpus_dir=corpus_dir, + ) + seen.add(case.id) + cases.append(case) + + return yaml.safe_dump( + {"cases": [_case_item(case) for case in cases]}, + sort_keys=False, + allow_unicode=False, + ) + + +def export_cases_from_db( + db: Session, + *, + case_ids: Sequence[str] | None = None, + dataset_path: Path | str = DEFAULT_DATASET, + corpus_dir: Path | str | None = CORPUS_DIR, +) -> str: + records = load_export_records(db, case_ids=case_ids) + return export_cases_fragment(records, dataset_path=dataset_path, corpus_dir=corpus_dir) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m app.eval.export_cases", + description="Export durable eval_cases rows as a reviewed YAML fragment.", + ) + parser.add_argument("--case-id", action="append", dest="case_ids", help="export one case id") + parser.add_argument("--dataset", default=str(DEFAULT_DATASET), help="fixed eval dataset path") + parser.add_argument("--corpus", default=str(CORPUS_DIR), help="fixed eval corpus directory") + parser.add_argument( + "--output", + default="-", + help="output file path; '-' writes to stdout", + ) + args = parser.parse_args(argv) + + from app.db.session import SessionLocal + + with SessionLocal() as db: + fragment = export_cases_from_db( + db, + case_ids=args.case_ids, + dataset_path=Path(args.dataset), + corpus_dir=Path(args.corpus), + ) + + if args.output == "-": + print(fragment, end="") + else: + Path(args.output).write_text(fragment, encoding="utf-8") + return 0 + + +if __name__ == "__main__": # pragma: no cover - CLI boundary + raise SystemExit(main()) diff --git a/backend/app/eval/harness.py b/backend/app/eval/harness.py index c0d23eb..eb63fe0 100644 --- a/backend/app/eval/harness.py +++ b/backend/app/eval/harness.py @@ -39,7 +39,8 @@ def run_eval(db: Session, embedder, dataset: list[EvalCase], config: EvalConfig, rows: list[dict] = [] for case in dataset: result = answer_question(db, embedder, client, cfg_settings, case.question, - top_k=config.top_k, source_ids=source_ids) + top_k=config.top_k, source_ids=source_ids, + agentic=config.agentic) if case.expect_refusal: hit = recall = reciprocal = None # retrieval metrics N/A for refusal cases keyword = None diff --git a/backend/app/eval/metrics.py b/backend/app/eval/metrics.py index 62ecbf2..8bb597f 100644 --- a/backend/app/eval/metrics.py +++ b/backend/app/eval/metrics.py @@ -6,10 +6,9 @@ """ from __future__ import annotations -import re from statistics import mean -_MARKER = re.compile(r"\[(\d+)\]") +from app.chat.prompt import all_citation_markers # --- retrieval metrics (retrieved/expected are document-title lists; retrieved is rank order) --- @@ -38,7 +37,7 @@ def mrr(retrieved: list[str], expected: list[str]) -> float: # --- answer-quality metrics (over the generated answer text) --- def extract_markers(answer: str) -> list[int]: - return [int(m) for m in _MARKER.findall(answer)] + return all_citation_markers(answer) def citation_validity(answer: str, n_context: int) -> float: diff --git a/backend/app/eval/pipeline.py b/backend/app/eval/pipeline.py index 4aab850..af9e01e 100644 --- a/backend/app/eval/pipeline.py +++ b/backend/app/eval/pipeline.py @@ -11,6 +11,7 @@ from sqlalchemy.orm import Session +from app.agentic_rag.service import answer_agentic_question from app.chat.prompt import ContextItem, build_messages, get_prompt from app.config import Settings from app.retrieval.hybrid import hybrid_search, load_display_chunks @@ -27,8 +28,29 @@ class AnswerResult: def answer_question(db: Session, embedder, llm, settings: Settings, question: str, - *, top_k: int | None = None, source_ids: list[int] | None = None + *, top_k: int | None = None, source_ids: list[int] | None = None, + agentic: bool = False, ) -> AnswerResult: + if agentic: + result = answer_agentic_question( + db, + embedder, + llm, + settings, + question, + top_k=top_k, + filters={"source_ids": source_ids} if source_ids else None, + ) + seen: set[str] = set() + retrieved_docs: list[str] = [] + for hit in result.hits: + title = result.display[hit.chunk_id].document_title + if title not in seen: + seen.add(title) + retrieved_docs.append(title) + return AnswerResult(result.answer, retrieved_docs, result.n_context, + result.latency_ms, result.model) + retrieval_query, _rewrite_meta = maybe_rewrite_query(llm, settings, question) hits, _meta = hybrid_search(db, embedder, settings, retrieval_query, top_k=top_k, source_ids=source_ids) diff --git a/backend/app/ingest/service.py b/backend/app/ingest/service.py index 47938cd..56f000e 100644 --- a/backend/app/ingest/service.py +++ b/backend/app/ingest/service.py @@ -80,6 +80,7 @@ def ingest_documents( documents: list[DocumentInput], settings: Settings | None = None, redis_client=None, + commit: bool = True, ) -> IngestResult: cfg = settings or default_settings src = _get_or_create_source(db, source) @@ -138,7 +139,8 @@ def ingest_documents( except Exception as exc: results.append(DocumentResult(None, doc_in.title, "failed", chash, error=str(exc))) - db.commit() - if any(d.status == "embedded" for d in results): + if commit: + db.commit() + if commit and any(d.status == "embedded" for d in results): bump_search_cache_epoch(redis_client, cfg) return IngestResult(source_id=src.id, documents=results) diff --git a/backend/app/jobs/handlers.py b/backend/app/jobs/handlers.py index f05f263..f5924f5 100644 --- a/backend/app/jobs/handlers.py +++ b/backend/app/jobs/handlers.py @@ -55,6 +55,7 @@ def handle_research(db: Session, payload: dict, *, embedder, llm) -> dict: payload.get("topic", ""), source_urls=payload.get("source_urls"), source_texts=payload.get("source_texts"), + commit=False, ) return { "topic": res.topic, diff --git a/backend/app/jobs/worker.py b/backend/app/jobs/worker.py index c798a56..a2272d8 100644 --- a/backend/app/jobs/worker.py +++ b/backend/app/jobs/worker.py @@ -17,6 +17,8 @@ from sqlalchemy.orm import Session +from app.cache.redis_client import get_redis_client +from app.cache.search import bump_search_cache_epoch from app.config import settings from app.jobs import queue from app.jobs.handlers import HANDLERS @@ -30,39 +32,43 @@ def run_once( max_attempts: int, handlers: dict[str, Callable] | None = None, types: Sequence[str] | None = None, + redis_client=None, + cache_settings=None, ): """Claim and process a single job. Returns the Job (done/failed) or None if the queue is empty. Any handler exception is recorded on the job (`mark_failed`) rather than raised. - One attempt is atomic: the claim (queued -> running) is committed first, then the handler - runs; on failure its partial writes are rolled back before the failure is recorded, so a - failed job never leaves orphaned rows (e.g. a half-written Briefing) behind. + One attempt is atomic: the claim (queued -> running) is flushed before the handler runs, + and the final done/failed state commits once at the end. On failure, handler writes are + rolled back before the failure is recorded, so a failed job never leaves orphaned rows. """ registry = HANDLERS if handlers is None else handlers + cfg = cache_settings or settings job = queue.claim_next(db, types=types) if job is None: return None handler = registry.get(job.type) + invalidate_search = False # SAVEPOINT around the handler: on failure its partial writes roll back while the claim # (queued -> running, taken before the savepoint) survives, so a failed job never commits - # orphaned rows (e.g. a half-written Briefing) alongside the failure record. Managed - # manually (not `with`) so a handler that commits internally — research via - # ingest_documents — is tolerated: that commit releases the savepoint, and is_active is - # then False, so we don't double-release it. + # orphaned rows (e.g. a half-written Briefing or research note) alongside the failure record. savepoint = db.begin_nested() try: if handler is None: raise LookupError(f"no handler registered for job type {job.type!r}") result = handler(db, job.payload, embedder=embedder, llm=llm) + queue.mark_done(db, job, result=result) if savepoint.is_active: savepoint.commit() - queue.mark_done(db, job, result=result) + invalidate_search = isinstance(result, dict) and bool(result.get("searchable")) except Exception as exc: # noqa: BLE001 — any handler failure is recorded on the job row if savepoint.is_active: savepoint.rollback() queue.mark_failed(db, job, str(exc), max_attempts=max_attempts) db.commit() + if invalidate_search: + bump_search_cache_epoch(redis_client, cfg) return job @@ -74,6 +80,8 @@ def run_loop( max_attempts: int, poll_seconds: float, types: Sequence[str] | None = None, + redis_client=None, + cache_settings=None, ) -> None: # pragma: no cover - resident deploy process, not unit-tested (sharp edge #2) """Poll the queue forever: drain eligible jobs, then sleep ``poll_seconds`` when idle. @@ -81,7 +89,15 @@ def run_loop( """ while True: with session_factory() as db: - job = run_once(db, embedder=embedder, llm=llm, max_attempts=max_attempts, types=types) + job = run_once( + db, + embedder=embedder, + llm=llm, + max_attempts=max_attempts, + types=types, + redis_client=redis_client, + cache_settings=cache_settings, + ) if job is None: time.sleep(poll_seconds) @@ -99,17 +115,25 @@ def main(argv: Sequence[str] | None = None) -> None: # pragma: no cover - CLI w embedder = get_embedder() llm = get_llm_client(settings) + redis_client = get_redis_client(settings) if args.loop: run_loop( SessionLocal, embedder=embedder, llm=llm, max_attempts=settings.job_max_attempts, poll_seconds=settings.worker_poll_seconds, + redis_client=redis_client, + cache_settings=settings, ) else: with SessionLocal() as db: job = run_once( - db, embedder=embedder, llm=llm, max_attempts=settings.job_max_attempts + db, + embedder=embedder, + llm=llm, + max_attempts=settings.job_max_attempts, + redis_client=redis_client, + cache_settings=settings, ) if job is None: print("no eligible job") diff --git a/backend/app/main.py b/backend/app/main.py index 3f31b98..d29314f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -23,7 +23,7 @@ CORSMiddleware, allow_origins=settings.cors_origins, allow_methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"], - allow_headers=["Authorization", "Content-Type"], + allow_headers=["Authorization", "Content-Type", "X-Second-Brain-Admin-Token"], allow_credentials=False, ) if settings.metrics_enabled: diff --git a/backend/app/research/service.py b/backend/app/research/service.py index 1d0a375..3af5f44 100644 --- a/backend/app/research/service.py +++ b/backend/app/research/service.py @@ -485,6 +485,7 @@ def research_topic( *, source_urls: Sequence[str] | None = None, source_texts: Sequence[SourceTextLike] | None = None, + commit: bool = True, ) -> ResearchResult: topic = (topic or "").strip() if not topic: @@ -510,6 +511,7 @@ def research_topic( "sources": source_metadata, }, )], + commit=commit, ) doc = result.documents[0] return ResearchResult( diff --git a/backend/app/schemas/chat.py b/backend/app/schemas/chat.py index 7115a73..7fa617c 100644 --- a/backend/app/schemas/chat.py +++ b/backend/app/schemas/chat.py @@ -12,6 +12,7 @@ class ChatFilters(BaseModel): class ChatOptions(BaseModel): private_mode: bool = False include_chunks: bool = True + agentic: bool = False class ChatRequest(BaseModel): diff --git a/backend/migrations/versions/0005_eval_cases.py b/backend/migrations/versions/0005_eval_cases.py new file mode 100644 index 0000000..bd41e0a --- /dev/null +++ b/backend/migrations/versions/0005_eval_cases.py @@ -0,0 +1,49 @@ +"""durable reviewed eval cases + +Store feedback-promoted eval cases in Postgres instead of mutating the source-controlled +dataset file from the production API container. + +Revision ID: 0005_eval_cases +Revises: 0004_briefings +Create Date: 2026-06-05 +""" +from __future__ import annotations + +from typing import Sequence, Union + +from alembic import op + +revision: str = "0005_eval_cases" +down_revision: Union[str, None] = "0004_briefings" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE eval_cases ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + case_id text NOT NULL, + feedback_id bigint REFERENCES feedback(id) ON DELETE SET NULL, + question text NOT NULL, + expected_docs jsonb NOT NULL DEFAULT '[]'::jsonb, + expected_keywords jsonb NOT NULL DEFAULT '[]'::jsonb, + expect_refusal boolean NOT NULL DEFAULT false, + review jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT uq_eval_cases_case_id UNIQUE (case_id) + ) + """ + ) + op.execute("CREATE INDEX ix_eval_cases_feedback_id ON eval_cases (feedback_id)") + op.execute("ALTER TABLE eval_cases ENABLE ROW LEVEL SECURITY") + op.execute( + "CREATE POLICY eval_cases_app_access ON eval_cases " + "USING (true) WITH CHECK (true)" + ) + + +def downgrade() -> None: + op.execute("DROP POLICY IF EXISTS eval_cases_app_access ON eval_cases") + op.execute("DROP TABLE IF EXISTS eval_cases") diff --git a/backend/requirements.prod.txt b/backend/requirements.prod.txt index 5eb3af8..0588d0a 100644 --- a/backend/requirements.prod.txt +++ b/backend/requirements.prod.txt @@ -16,6 +16,7 @@ sentence-transformers>=5.5,<6 transformers>=5.10.2,<6 google-genai>=0.3,<2 httpx>=0.27,<1 +langgraph>=1.0,<2 prometheus-client>=0.20,<1 redis>=5,<6 diff --git a/backend/requirements.txt b/backend/requirements.txt index 0483255..9ae44fe 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -24,6 +24,7 @@ pyyaml>=6,<7 # eval dataset (eval/dataset.yaml) # Phase 4 — MCP server + agentic actions mcp>=1.2,<2 # Model Context Protocol server (FastMCP, stdio) exposing the agent tools +langgraph>=1.0,<2 # Low-level graph orchestration for opt-in agentic RAG # Phase 6 — productionization + observability prometheus-client>=0.20,<1 # /metrics exposition for self-hosted Prometheus diff --git a/backend/tests/integration/test_api.py b/backend/tests/integration/test_api.py index 04e78cd..e3a7d0a 100644 --- a/backend/tests/integration/test_api.py +++ b/backend/tests/integration/test_api.py @@ -58,6 +58,46 @@ def test_chat_empty_corpus(client): assert body["model"] is None +def test_agentic_chat_disabled_by_default(client): + r = client.post("/chat", json={ + "message": "anything?", + "options": {"agentic": True}, + }) + assert r.status_code == 409 + assert r.json()["detail"] == "agentic RAG is disabled" + + +def test_agentic_chat_endpoint_returns_trace_when_enabled(client, test_settings): + test_settings.agentic_rag_enabled = True + ing = client.post("/ingest", json={ + "source": {"type": "manual", "name": "Agentic API Notes"}, + "documents": [{"title": "Agentic HNSW", + "content": "HNSW tuning m ef_construction. " * 20}], + }) + assert ing.status_code == 200, ing.text + + r = client.post("/chat", json={ + "message": "How do I tune HNSW?", + "options": {"agentic": True}, + }) + + assert r.status_code == 200, r.text + body = r.json() + assert body["citations"] + assert body["retrieval"]["method"] == "agentic_hybrid" + assert body["retrieval"]["agentic"]["enabled"] is True + assert body["retrieval"]["agentic"]["subqueries"] + + +def test_agentic_chat_stream_returns_unavailable(client): + r = client.post("/chat/stream", json={ + "message": "anything?", + "options": {"agentic": True}, + }) + assert r.status_code == 409 + assert r.json()["detail"] == "streaming is unavailable for agentic RAG" + + def test_chat_stream_sends_sse_deltas_and_completion(client): ing = client.post("/ingest", json={ "source": {"type": "manual", "name": "Streaming Notes"}, diff --git a/backend/tests/integration/test_chat.py b/backend/tests/integration/test_chat.py index 5442e2f..dda866c 100644 --- a/backend/tests/integration/test_chat.py +++ b/backend/tests/integration/test_chat.py @@ -23,7 +23,14 @@ def test_chat_persists_and_cites(db_session, fake_embedder): def test_chat_empty_corpus_refuses(db_session, fake_embedder): - r = chat(db_session, fake_embedder, FakeLLMClient(), Settings(), message="anything?") + r = chat( + db_session, + fake_embedder, + FakeLLMClient(), + Settings(), + message="anything?", + filters={"source_ids": [-1]}, + ) assert r.citations == [] and r.model is None @@ -52,6 +59,28 @@ def generate(self, messages: list[LLMMessage]) -> LLMResponse: return LLMResponse(text="The moon is made of cheese [1].", model=self.model) +class _RepairingCitationLLM: + model = "repairing-citation-fake" + + def __init__(self): + self.calls = 0 + + def generate(self, messages: list[LLMMessage]) -> LLMResponse: + self.calls += 1 + if self.calls == 1: + return LLMResponse( + text=( + "Here is what the notes say. " + "Hybrid retrieval combines vector search and full-text search [1]." + ), + model=self.model, + ) + return LLMResponse( + text="Hybrid retrieval combines vector search and full-text search [1].", + model=self.model, + ) + + class _LeakyStreamingLLM: model = "leaky-stream-fake" @@ -114,6 +143,27 @@ def test_chat_replaces_unsupported_cited_answer_with_citation_failure(db_session assert r.retrieval["unsupported_citation_segments"] +def test_chat_repairs_uncited_framing_before_persisting(db_session, fake_embedder): + ingest_documents(db_session, fake_embedder, source=SourceSpec("manual", "Repair"), + documents=[DocumentInput(title="Hybrid retrieval", + content=( + "Hybrid retrieval combines vector search " + "and full-text search. " + ) * 20)]) + llm = _RepairingCitationLLM() + r = chat(db_session, fake_embedder, llm, Settings(), + message="What does hybrid retrieval combine?") + + assert llm.calls == 2 + assert r.answer == "Hybrid retrieval combines vector search and full-text search [1]." + assert r.citations + assert r.retrieval["citation_repair_attempted"] is True + assert r.retrieval["citation_repair_succeeded"] is True + stored = db_session.get(Message, r.message_id) + assert stored is not None + assert stored.content == r.answer + + def test_chat_continues_conversation(db_session, fake_embedder): ingest_documents(db_session, fake_embedder, source=SourceSpec("manual", "Conv"), documents=[DocumentInput(title="Doc", @@ -126,6 +176,51 @@ def test_chat_continues_conversation(db_session, fake_embedder): assert r2.message_id != r1.message_id +def test_agentic_chat_persists_cited_answer_with_trace(db_session, fake_embedder): + from app.agentic_rag.service import agentic_chat + + ingest_documents(db_session, fake_embedder, source=SourceSpec("manual", "Agentic"), + documents=[DocumentInput(title="Agentic HNSW", + content="HNSW tuning m ef_construction. " * 20)]) + + r = agentic_chat( + db_session, + fake_embedder, + FakeLLMClient(), + Settings(agentic_rag_enabled=True), + message="What about HNSW tuning?", + ) + + assert r.message_id and r.conversation_id + assert r.citations + assert r.retrieval["method"] == "agentic_hybrid" + assert r.retrieval["agentic"]["enabled"] is True + assert r.retrieval["agentic"]["subqueries"] + assert r.retrieval["agentic"]["selected_chunks"] >= 1 + stored = db_session.get(Message, r.message_id) + assert stored is not None + assert stored.content == r.answer + + +def test_agentic_chat_empty_corpus_refuses_with_trace(db_session, fake_embedder): + from app.agentic_rag.service import agentic_chat + + r = agentic_chat( + db_session, + fake_embedder, + FakeLLMClient(), + Settings(agentic_rag_enabled=True), + message="anything?", + filters={"source_ids": [-1]}, + ) + + assert r.citations == [] + assert r.model is None + assert r.retrieval["refusal_reason"] == "weak_context" + assert r.retrieval["agentic"]["enabled"] is True + assert r.retrieval["agentic"]["weak_evidence"] is True + + def test_stream_chat_emits_deltas_and_persists_cited_completion(db_session, fake_embedder): ingest_documents(db_session, fake_embedder, source=SourceSpec("manual", "Stream"), documents=[DocumentInput(title="Streaming", diff --git a/backend/tests/integration/test_demo_seed.py b/backend/tests/integration/test_demo_seed.py new file mode 100644 index 0000000..0c91de4 --- /dev/null +++ b/backend/tests/integration/test_demo_seed.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from sqlalchemy import select + +from app.config import Settings +from app.db.models import Feedback, Source +from app.demo.seed import DEMO_QUESTION, seed_demo_flow +from app.llm.fake import FakeLLMClient + + +def test_seed_demo_flow_creates_capture_chat_and_feedback(db_session, fake_embedder): + result = seed_demo_flow( + db_session, + fake_embedder, + FakeLLMClient(), + Settings(llm_provider="fake"), + ) + + source = db_session.get(Source, result.source_id) + feedback = db_session.get(Feedback, result.feedback_id) + + assert source is not None + assert source.type == "bookmark" + assert result.document_id is not None + assert result.question == DEMO_QUESTION + assert feedback is not None + assert feedback.rating == -1 + assert feedback.message_id == result.assistant_message_id + assert db_session.scalar(select(Feedback).where(Feedback.id == result.feedback_id)) is not None diff --git a/backend/tests/integration/test_jobs_worker.py b/backend/tests/integration/test_jobs_worker.py index 9bb8adb..ad3a363 100644 --- a/backend/tests/integration/test_jobs_worker.py +++ b/backend/tests/integration/test_jobs_worker.py @@ -8,7 +8,9 @@ from sqlalchemy import select +from app.config import Settings from app.db.models import Source +from app.ingest.service import DocumentInput, SourceSpec, ingest_documents from app.jobs import handlers, queue, worker from app.llm.fake import FakeLLMClient @@ -73,6 +75,97 @@ def writes_then_raises(db, payload, *, embedder, llm): assert db_session.scalar(select(Source).where(Source.name == marker)) is None +def test_run_once_rolls_back_non_committing_ingest_on_failure(db_session, fake_embedder): + marker = "ATOMIC_RESEARCH_STYLE_INGEST" + + def ingests_then_raises(db, payload, *, embedder, llm): + ingest_documents( + db, + embedder, + source=SourceSpec(type="research_note", name=marker), + documents=[DocumentInput(title="staged research", content="worker staged content")], + commit=False, + ) + raise RuntimeError("boom after staged ingest") + + queue.enqueue(db_session, type="embed") + job = worker.run_once( + db_session, + embedder=fake_embedder, + llm=FakeLLMClient(), + max_attempts=1, + handlers={"embed": ingests_then_raises}, + ) + + assert job is not None and job.status == "failed" + assert db_session.scalar(select(Source).where(Source.name == marker)) is None + + +def test_run_once_invalidates_search_cache_after_searchable_job(db_session, fake_embedder): + class FakeRedis: + def __init__(self): + self.keys: list[str] = [] + + def incr(self, key): + self.keys.append(key) + + redis = FakeRedis() + + def searchable_handler(db, payload, *, embedder, llm): + return {"searchable": True} + + queue.enqueue(db_session, type="embed") + job = worker.run_once( + db_session, + embedder=fake_embedder, + llm=FakeLLMClient(), + max_attempts=1, + handlers={"embed": searchable_handler}, + redis_client=redis, + cache_settings=Settings(llm_provider="fake", redis_enabled=True), + ) + + assert job is not None and job.status == "done" + assert redis.keys == ["cache:search:epoch"] + + +def test_run_once_does_not_invalidate_search_cache_when_mark_done_fails( + db_session, + fake_embedder, + monkeypatch, +): + class FakeRedis: + def __init__(self): + self.keys: list[str] = [] + + def incr(self, key): + self.keys.append(key) + + def searchable_handler(db, payload, *, embedder, llm): + return {"searchable": True} + + def failing_mark_done(db, job, result=None): + raise RuntimeError("mark_done failed") + + redis = FakeRedis() + queue.enqueue(db_session, type="embed") + monkeypatch.setattr(queue, "mark_done", failing_mark_done) + + job = worker.run_once( + db_session, + embedder=fake_embedder, + llm=FakeLLMClient(), + max_attempts=1, + handlers={"embed": searchable_handler}, + redis_client=redis, + cache_settings=Settings(llm_provider="fake", redis_enabled=True), + ) + + assert job is not None and job.status == "failed" + assert "mark_done failed" in (job.last_error or "") + assert redis.keys == [] + + def test_run_once_returns_none_when_no_job(db_session, fake_embedder): job = worker.run_once( db_session, embedder=fake_embedder, llm=FakeLLMClient(), diff --git a/backend/tests/integration/test_rls.py b/backend/tests/integration/test_rls.py index 975a647..ed3ab22 100644 --- a/backend/tests/integration/test_rls.py +++ b/backend/tests/integration/test_rls.py @@ -18,6 +18,7 @@ "messages", "retrievals", "feedback", + "eval_cases", ] diff --git a/backend/tests/integration/test_search.py b/backend/tests/integration/test_search.py index 8773612..acbdd09 100644 --- a/backend/tests/integration/test_search.py +++ b/backend/tests/integration/test_search.py @@ -6,8 +6,7 @@ from app import deps from app.api import conversations from app.config import Settings -from app.db.models import AuditLog -from app.eval.dataset import load_dataset +from app.db.models import AuditLog, EvalCaseRecord from app.main import app pytestmark = pytest.mark.skipif( @@ -303,7 +302,7 @@ def test_promote_feedback_eval_candidate_requires_admin_token(client, monkeypatc assert dataset.read_text(encoding="utf-8") == "cases: []\n" -def test_promote_feedback_eval_candidate_appends_reviewed_case( +def test_promote_feedback_eval_candidate_persists_reviewed_case( client, db_session, monkeypatch, tmp_path ): _enable_admin() @@ -333,29 +332,86 @@ def test_promote_feedback_eval_candidate_appends_reviewed_case( assert r.status_code == 201, r.text body = r.json() - assert body["dataset_path"] == "backend/eval/dataset.yaml" + assert body["dataset_path"] == "postgres:eval_cases" assert body["case"]["id"] == f"feedback-{fb['id']}-reviewed" assert body["case"]["metadata"]["needs_review"] is False - cases = load_dataset(dataset, corpus_dir=corpus) - assert cases[0].id == body["case"]["id"] - assert cases[0].expected_docs == ["Feedback analytics doc"] - assert cases[0].expected_keywords == ["sources"] - assert cases[0].review["source"] == "feedback" - assert cases[0].review["feedback_id"] == fb["id"] - assert cases[0].review["confirmations"] == { + assert body["case"]["metadata"]["storage"] == "postgres" + assert dataset.read_text(encoding="utf-8") == "cases: []\n" + + stored = db_session.scalars( + select(EvalCaseRecord).where(EvalCaseRecord.case_id == body["case"]["id"]) + ).one() + assert stored.question == "What should feedback review connect thumbs down answers to?" + assert stored.expected_docs == ["Feedback analytics doc"] + assert stored.expected_keywords == ["sources"] + assert stored.review["source"] == "feedback" + assert stored.review["feedback_id"] == fb["id"] + assert stored.review["confirmations"] == { "expect_refusal": True, "expected_docs": True, "expected_keywords": True, } - assert body["case"]["metadata"]["review"] == cases[0].review + assert body["case"]["metadata"]["eval_case_record_id"] == stored.id + assert body["case"]["metadata"]["review"] == stored.review audit_row = db_session.scalars( select(AuditLog).where(AuditLog.entity_type == "eval_case") ).one() assert audit_row.action == "create" - assert audit_row.entity_id == fb["id"] + assert audit_row.entity_id == stored.id assert audit_row.detail["op"] == "promote_eval_case" + assert audit_row.detail["storage"] == "postgres" + assert audit_row.detail["eval_case_record_id"] == stored.id assert audit_row.detail["case_id"] == body["case"]["id"] - assert audit_row.detail["review"] == cases[0].review + assert audit_row.detail["feedback_id"] == fb["id"] + assert audit_row.detail["review"] == stored.review + + +def test_promote_feedback_eval_candidate_returns_conflict_on_duplicate_race( + client, db_session, monkeypatch, tmp_path +): + _enable_admin() + chat = _seed_feedback_with_cited_answer(client) + fb = client.post( + "/feedback", + json={"message_id": chat["message_id"], "rating": -1, "comment": "needs eval"}, + ).json() + dataset, _corpus = _patch_review_dataset(monkeypatch, tmp_path) + case_id = f"feedback-{fb['id']}-reviewed" + + db_session.add( + EvalCaseRecord( + case_id=case_id, + feedback_id=fb["id"], + question="Existing race winner", + expected_docs=["Feedback analytics doc"], + expected_keywords=["sources"], + expect_refusal=False, + review={}, + ) + ) + db_session.flush() + monkeypatch.setattr(conversations, "validate_new_eval_case", lambda case, **kwargs: case) + + r = client.post( + f"/feedback/eval-candidates/{fb['id']}/promote", + json={ + "id": case_id, + "question": "What should feedback review connect thumbs down answers to?", + "expected_docs": ["Feedback analytics doc"], + "expected_keywords": ["sources"], + "expect_refusal": False, + "confirmations": { + "expected_docs": True, + "expected_keywords": True, + "expect_refusal": True, + }, + }, + headers=ADMIN, + ) + + assert r.status_code == 409 + assert "already exists" in r.text + assert dataset.read_text(encoding="utf-8") == "cases: []\n" def test_promote_feedback_eval_candidate_rejects_unknown_expected_doc( diff --git a/backend/tests/unit/test_agentic_rag.py b/backend/tests/unit/test_agentic_rag.py new file mode 100644 index 0000000..63bfe9f --- /dev/null +++ b/backend/tests/unit/test_agentic_rag.py @@ -0,0 +1,37 @@ +from app.agentic_rag.service import parse_query_plan + + +def test_parse_query_plan_accepts_json_object_and_clamps(): + queries, failed = parse_query_plan( + '{"queries":[" HNSW tuning ","ef construction","HNSW tuning","extra"]}', + question="What about HNSW?", + max_queries=2, + max_chars=80, + ) + + assert queries == ["HNSW tuning", "ef construction"] + assert failed is False + + +def test_parse_query_plan_accepts_fenced_json_object(): + queries, failed = parse_query_plan( + '```json\n{"queries":["hybrid retrieval","eval gating","citation validation"]}\n```', + question="How do safety checks work?", + max_queries=3, + max_chars=80, + ) + + assert queries == ["hybrid retrieval", "eval gating", "citation validation"] + assert failed is False + + +def test_parse_query_plan_falls_back_to_original_question(): + queries, failed = parse_query_plan( + "not json", + question="What about hybrid retrieval?", + max_queries=4, + max_chars=80, + ) + + assert "What about hybrid retrieval?" in queries + assert failed is True diff --git a/backend/tests/unit/test_api_auth.py b/backend/tests/unit/test_api_auth.py index 887900e..2d4675c 100644 --- a/backend/tests/unit/test_api_auth.py +++ b/backend/tests/unit/test_api_auth.py @@ -103,6 +103,33 @@ def test_authenticated_request_passes_api_gate_but_still_validates_request_body( app.dependency_overrides.clear() +def test_cors_preflight_allows_admin_token_header(): + app.dependency_overrides[deps.get_settings] = lambda: Settings( + _env_file=None, + cors_origins=["http://localhost:3000"], + metrics_enabled=False, + ) + try: + with TestClient(app) as client: + response = client.options( + "/data/export?source_id=1", + headers={ + "Origin": "http://localhost:3000", + "Access-Control-Request-Method": "GET", + "Access-Control-Request-Headers": ( + "Authorization, X-Second-Brain-Admin-Token" + ), + }, + ) + + assert response.status_code == 200 + allowed = response.headers["access-control-allow-headers"].lower() + assert "authorization" in allowed + assert "x-second-brain-admin-token" in allowed + finally: + app.dependency_overrides.clear() + + def test_destructive_dataops_requires_admin_after_api_gate(monkeypatch): class DummyDb: committed = False diff --git a/backend/tests/unit/test_chat_citation_support.py b/backend/tests/unit/test_chat_citation_support.py new file mode 100644 index 0000000..c1b07e1 --- /dev/null +++ b/backend/tests/unit/test_chat_citation_support.py @@ -0,0 +1,47 @@ +from app.chat.service import _PreparedChat, _citation_support_failures +from app.llm.base import LLMMessage +from app.retrieval.fusion import FusedHit +from app.retrieval.hybrid import DisplayChunk + + +def test_grouped_citation_markers_support_claim_segment(): + prepared = _PreparedChat( + conversation_id=1, + messages=[LLMMessage("user", "q")], + hits=[ + FusedHit(chunk_id=10, score=1.0, method="hybrid", rank=1), + FusedHit(chunk_id=11, score=0.9, method="hybrid", rank=2), + ], + display={ + 10: DisplayChunk( + chunk_id=10, + content="Hybrid retrieval combines vector search and full-text search.", + document_id=1, + document_title="Hybrid Retrieval", + source_id=1, + source_name="Docs", + char_start=0, + char_end=64, + ), + 11: DisplayChunk( + chunk_id=11, + content="Eval gating and citation validation make generated answers safer.", + document_id=2, + document_title="Eval Safety", + source_id=1, + source_name="Docs", + char_start=0, + char_end=66, + ), + }, + meta={}, + item_count=2, + include_chunks=True, + ) + + failures = _citation_support_failures( + "Hybrid retrieval, eval gating, and citation validation make answers safer [1, 2].", + prepared, + ) + + assert failures == [] diff --git a/backend/tests/unit/test_config.py b/backend/tests/unit/test_config.py index 45f0c4d..05835ad 100644 --- a/backend/tests/unit/test_config.py +++ b/backend/tests/unit/test_config.py @@ -13,7 +13,11 @@ def test_defaults(monkeypatch): "SECOND_BRAIN_API_TOKEN", "SECOND_BRAIN_RATE_LIMIT_FAIL_CLOSED", "SECOND_BRAIN_TRUST_FORWARDED_FOR", - "SECOND_BRAIN_MCP_ENABLE_MUTATIONS"]: + "SECOND_BRAIN_MCP_ENABLE_MUTATIONS", + "SECOND_BRAIN_AGENTIC_RAG_ENABLED", + "SECOND_BRAIN_AGENTIC_RAG_MAX_SUBQUERIES", + "SECOND_BRAIN_AGENTIC_RAG_VERIFIER_ENABLED", + "SECOND_BRAIN_AGENTIC_RAG_RECURSION_LIMIT"]: monkeypatch.delenv(key, raising=False) s = Settings(_env_file=None) assert s.llm_provider == "gemini" @@ -23,10 +27,15 @@ def test_defaults(monkeypatch): assert s.retrieval_query_rewrite_enabled is False assert s.prompt_version == "rag-v1" assert s.mlflow_tracking_uri == "file:./mlruns" + assert s.cors_origins == ["http://localhost:3000", "http://127.0.0.1:3000"] assert s.api_token is None assert s.rate_limit_fail_closed is True assert s.trust_forwarded_for is False assert s.mcp_enable_mutations is False + assert s.agentic_rag_enabled is False + assert s.agentic_rag_max_subqueries == 4 + assert s.agentic_rag_verifier_enabled is True + assert s.agentic_rag_recursion_limit == 8 def test_env_override(monkeypatch): @@ -38,6 +47,10 @@ def test_env_override(monkeypatch): monkeypatch.setenv("SECOND_BRAIN_RATE_LIMIT_FAIL_CLOSED", "false") monkeypatch.setenv("SECOND_BRAIN_TRUST_FORWARDED_FOR", "true") monkeypatch.setenv("SECOND_BRAIN_MCP_ENABLE_MUTATIONS", "true") + monkeypatch.setenv("SECOND_BRAIN_AGENTIC_RAG_ENABLED", "true") + monkeypatch.setenv("SECOND_BRAIN_AGENTIC_RAG_MAX_SUBQUERIES", "3") + monkeypatch.setenv("SECOND_BRAIN_AGENTIC_RAG_VERIFIER_ENABLED", "false") + monkeypatch.setenv("SECOND_BRAIN_AGENTIC_RAG_RECURSION_LIMIT", "6") s = Settings() assert s.llm_provider == "fake" assert s.retrieval_top_k == 3 @@ -47,3 +60,7 @@ def test_env_override(monkeypatch): assert s.rate_limit_fail_closed is False assert s.trust_forwarded_for is True assert s.mcp_enable_mutations is True + assert s.agentic_rag_enabled is True + assert s.agentic_rag_max_subqueries == 3 + assert s.agentic_rag_verifier_enabled is False + assert s.agentic_rag_recursion_limit == 6 diff --git a/backend/tests/unit/test_eval_configs.py b/backend/tests/unit/test_eval_configs.py index cc8af62..aef7364 100644 --- a/backend/tests/unit/test_eval_configs.py +++ b/backend/tests/unit/test_eval_configs.py @@ -4,10 +4,11 @@ def test_registry_has_ab_and_real_configs(): - assert {"baseline", "variant", "gemini"} <= set(CONFIGS) + assert {"baseline", "variant", "agentic", "gemini", "gemini-agentic"} <= set(CONFIGS) assert CONFIGS["baseline"].prompt_version == "rag-v1" assert CONFIGS["baseline"].llm_provider == "fake" assert CONFIGS["variant"].prompt_version == "rag-v2" + assert CONFIGS["agentic"].agentic is True assert CONFIGS["gemini"].llm_provider == "gemini" @@ -16,6 +17,13 @@ def test_settings_for_applies_overrides(): assert s.llm_provider == "fake" assert s.prompt_version == "rag-v2" assert s.retrieval_top_k == 5 + assert s.agentic_rag_enabled is False + + +def test_agentic_config_enables_agentic_rag(): + s = settings_for(CONFIGS["agentic"], base=Settings(_env_file=None)) + assert s.llm_provider == "fake" + assert s.agentic_rag_enabled is True def test_default_ab_pair_is_single_variable(): diff --git a/backend/tests/unit/test_eval_export_cases.py b/backend/tests/unit/test_eval_export_cases.py new file mode 100644 index 0000000..ea1ae57 --- /dev/null +++ b/backend/tests/unit/test_eval_export_cases.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from app.eval.export_cases import export_cases_fragment, record_to_eval_case + + +def _review(feedback_id: int = 1) -> dict: + return { + "source": "feedback", + "feedback_id": feedback_id, + "reviewed_at": "2026-06-05T12:00:00+00:00", + "reviewed_by": "eval-reviewer", + "confirmations": { + "expected_docs": True, + "expected_keywords": True, + "expect_refusal": True, + }, + } + + +def _record(case_id: str, *, feedback_id: int = 1): + return SimpleNamespace( + case_id=case_id, + question="What should feedback review prove?", + expected_docs=["Feedback analytics doc"], + expected_keywords=["feedback"], + expect_refusal=False, + review=_review(feedback_id), + ) + + +def test_record_to_eval_case_preserves_reviewed_fields(): + case = record_to_eval_case(_record("feedback-1-reviewed")) + + assert case.id == "feedback-1-reviewed" + assert case.expected_docs == ["Feedback analytics doc"] + assert case.expected_keywords == ["feedback"] + assert case.review["feedback_id"] == 1 + + +def test_export_cases_fragment_skips_cases_already_in_fixed_dataset(tmp_path): + dataset = tmp_path / "dataset.yaml" + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "feedback.md").write_text("# Feedback analytics doc\nbody\n", encoding="utf-8") + dataset.write_text( + "cases:\n" + " - id: already-fixed\n" + " question: Existing?\n" + " expected_docs:\n" + " - Feedback analytics doc\n" + " expected_keywords:\n" + " - feedback\n" + " expect_refusal: false\n", + encoding="utf-8", + ) + + text = export_cases_fragment( + [_record("already-fixed"), _record("new-reviewed", feedback_id=2)], + dataset_path=dataset, + corpus_dir=corpus, + ) + + assert "already-fixed" not in text + assert "new-reviewed" in text + assert "review:" in text + assert "feedback_id: 2" in text + + +def test_export_cases_fragment_validates_against_fixed_corpus(tmp_path): + dataset = tmp_path / "dataset.yaml" + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "other.md").write_text("# Other doc\nbody\n", encoding="utf-8") + dataset.write_text("cases: []\n", encoding="utf-8") + + with pytest.raises(ValueError, match="not in the eval corpus"): + export_cases_fragment([_record("bad-doc")], dataset_path=dataset, corpus_dir=corpus) diff --git a/backend/tests/unit/test_eval_metrics.py b/backend/tests/unit/test_eval_metrics.py index df95050..7b009da 100644 --- a/backend/tests/unit/test_eval_metrics.py +++ b/backend/tests/unit/test_eval_metrics.py @@ -24,6 +24,7 @@ def test_mrr(): def test_citation_validity(): assert m.citation_validity("uses [1] and [2]", n_context=3) == 1.0 + assert m.citation_validity("uses grouped markers [1, 2]", n_context=3) == 1.0 assert m.citation_validity("uses [1] and [9]", n_context=3) == 0.5 # [9] out of range assert m.citation_validity("no markers here", n_context=3) == 1.0 # nothing hallucinated diff --git a/backend/tests/unit/test_prompt.py b/backend/tests/unit/test_prompt.py index 6f3a617..96370dd 100644 --- a/backend/tests/unit/test_prompt.py +++ b/backend/tests/unit/test_prompt.py @@ -14,3 +14,4 @@ def test_build_messages_numbers_context_and_includes_history(): def test_parse_citations_dedup_and_range(): assert parse_citations("uses [2] and [1] and [2] and [9]", n_items=3) == [2, 1] + assert parse_citations("uses grouped markers [2, 1, 2] and bad [9]", n_items=3) == [2, 1] diff --git a/deploy/Dockerfile.frontend b/deploy/Dockerfile.frontend index ef17709..4e469b5 100644 --- a/deploy/Dockerfile.frontend +++ b/deploy/Dockerfile.frontend @@ -21,6 +21,8 @@ COPY frontend/ ./ # through ingress. Additive + backwards-compatible. ARG NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 ENV NEXT_PUBLIC_API_BASE_URL=$NEXT_PUBLIC_API_BASE_URL +ARG NEXT_PUBLIC_AGENTIC_RAG_ENABLED=false +ENV NEXT_PUBLIC_AGENTIC_RAG_ENABLED=$NEXT_PUBLIC_AGENTIC_RAG_ENABLED RUN npm run build RUN npm prune --omit=dev --no-audit --no-fund \ diff --git a/deploy/docker-compose.prod.yml b/deploy/docker-compose.prod.yml index 8c03ad4..354c8da 100644 --- a/deploy/docker-compose.prod.yml +++ b/deploy/docker-compose.prod.yml @@ -43,6 +43,7 @@ services: SECOND_BRAIN_EMBEDDING_PROVIDER: ${SECOND_BRAIN_EMBEDDING_PROVIDER:-local} SECOND_BRAIN_API_TOKEN: ${SECOND_BRAIN_API_TOKEN:?set SECOND_BRAIN_API_TOKEN} SECOND_BRAIN_ADMIN_TOKEN: ${SECOND_BRAIN_ADMIN_TOKEN:-} + SECOND_BRAIN_AGENTIC_RAG_ENABLED: ${SECOND_BRAIN_AGENTIC_RAG_ENABLED:-false} SECOND_BRAIN_REDIS_ENABLED: ${SECOND_BRAIN_REDIS_ENABLED:-true} SECOND_BRAIN_REDIS_URL: ${SECOND_BRAIN_REDIS_URL:-redis://redis:6379/0} SECOND_BRAIN_RATE_LIMIT_FAIL_CLOSED: ${SECOND_BRAIN_RATE_LIMIT_FAIL_CLOSED:-true} @@ -80,8 +81,12 @@ services: build: context: .. dockerfile: deploy/Dockerfile.frontend + args: + NEXT_PUBLIC_API_BASE_URL: ${NEXT_PUBLIC_API_BASE_URL:-http://localhost:8000} + NEXT_PUBLIC_AGENTIC_RAG_ENABLED: ${NEXT_PUBLIC_AGENTIC_RAG_ENABLED:-false} environment: NEXT_PUBLIC_API_BASE_URL: ${NEXT_PUBLIC_API_BASE_URL:-http://localhost:8000} + NEXT_PUBLIC_AGENTIC_RAG_ENABLED: ${NEXT_PUBLIC_AGENTIC_RAG_ENABLED:-false} depends_on: - api ports: diff --git a/deploy/docker-compose.vps.yml.example b/deploy/docker-compose.vps.yml.example index a2fe161..9e0aa47 100644 --- a/deploy/docker-compose.vps.yml.example +++ b/deploy/docker-compose.vps.yml.example @@ -19,13 +19,14 @@ services: frontend: build: args: - NEXT_PUBLIC_API_BASE_URL: https://YOUR_VPS_IP.sslip.io/api + NEXT_PUBLIC_API_BASE_URL: ${NEXT_PUBLIC_API_BASE_URL:-https://YOUR_VPS_IP.sslip.io/api} + NEXT_PUBLIC_AGENTIC_RAG_ENABLED: ${NEXT_PUBLIC_AGENTIC_RAG_ENABLED:-false} ports: !override - "127.0.0.1:3000:3000" api: environment: - SECOND_BRAIN_CORS_ORIGINS: '["https://YOUR_VPS_IP.sslip.io"]' + SECOND_BRAIN_CORS_ORIGINS: ${SECOND_BRAIN_CORS_ORIGINS:-["https://YOUR_VPS_IP.sslip.io"]} ports: !override - "127.0.0.1:8000:8000" diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index 2b493c0..8ac2f27 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -1,7 +1,7 @@ # Kubernetes Learning Track (local kind) -> Kubernetes here is a learning track, not the production runtime. Production stays the single-VPS -> Docker Compose stack. These manifests prove the core app runs on local Kubernetes, then the +> Kubernetes here is a learning track, not the production runtime. The real app stays local-first +> Docker Compose by default. These manifests prove the core app runs on local Kubernetes, then the > cluster is torn down so nothing keeps running or costs money. The default `kubectl apply -k deploy/k8s` path runs the core stack only: `db`, migrations, `redis`, diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 5aa2b3d..1f0314f 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -14,7 +14,7 @@ session — the master prompt treats it as the source of truth for "where we are | 3 | Evaluation + MLOps: eval set, MLflow, A/B, prompt versioning + rollback | ✅ Complete | | 4 | MCP server + agentic actions incl. self-research tool | ✅ Complete | | 5 | Daily briefing + scheduled pipelines | ✅ Complete | -| 6 | Productionize on VPS + data-ops hardening | ✅ Complete | +| 6 | Operations hardening + optional cloud deploy recipe | ✅ Complete | | 7 | Kubernetes learning track on local k3s/kind | ✅ Complete | Legend: ⬜ not started · 🟡 in progress · ✅ complete @@ -23,6 +23,99 @@ Legend: ⬜ not started · 🟡 in progress · ✅ complete Add a dated entry per working session. Most recent on top. +### 2026-06-05 - PR review follow-up for Agentic RAG +- **What:** addressed the actionable CodeRabbit review comments on PR #23. Disabled agentic + requests now fail before LLM provider initialization, worker search-cache invalidation only + happens after successful job finalization, demo seeding guards empty ingests, and the chat UI + ignores stale non-stream responses after route aborts. +- **Docs/UI:** clarified optional VPS override setup and local-vs-proxy admin routes, fixed local + runtime wording, and added accessible state metadata to the Agentic RAG composer toggle. +- **Verified:** full backend suite passed (`261 passed, 8 warnings`); frontend `npm run lint` and + `npm run build` passed; `git diff --check` passed with only expected Windows CRLF notices. + +### 2026-06-05 - Agentic RAG v1 implemented +- **What:** added opt-in, read-only Agentic RAG for `/chat` behind + `SECOND_BRAIN_AGENTIC_RAG_ENABLED=true` plus request `options.agentic=true`. The LangGraph + request graph plans bounded subqueries, searches existing notes with the current hybrid retriever, + dedupes/merges evidence, optionally retries weak evidence with the original wording, and answers + through the existing citation/support finalizer. +- **Frontend/docs:** added optional web toggle support gated by + `NEXT_PUBLIC_AGENTIC_RAG_ENABLED=true`, compact answer-footer trace metadata, ADR-0016, usage + docs, env templates, and Compose build/runtime wiring for the false-by-default flags. +- **Local test fix:** expanded default dev CORS origins to include both `http://localhost:3000` + and `http://127.0.0.1:3000`, fixing browser `Failed to fetch` errors when the app is opened via + `127.0.0.1`. +- **Chat UI fix:** clicking **New Chat** from an existing `/chat?cid=...` conversation now resets + the mounted chat page state to a blank new conversation instead of keeping the old messages. +- **History menu fix:** the sidebar **Recent** list now collapses duplicate conversation titles + created by repeated test prompts, while keeping the active duplicate visible if it is open. +- **Citation reliability fix:** grouped citation markers like `[1, 2]` now validate correctly, + fenced planner JSON is parsed correctly, and failed cited drafts get one internal citation-repair + retry before the existing failure response is used. +- **Eval:** added `agentic` and `gemini-agentic` eval configs so the new path can be compared + against the regular RAG baseline before becoming default. +- **Verified:** focused unit tests passed (`9 passed`); focused chat/API integration tests passed + (`19 passed`); eval-focused tests passed (`24 passed, 2 skipped`); full backend suite passed + (`257 passed, 8 warnings`); eval runner `baseline,agentic --no-mlflow` completed with both at + `1.000` hit/recall/citation/refusal on the fake-driver set; frontend `npm run lint`, `npm run + build`, and `npm audit --audit-level=high` passed; frontend lint/build were re-run after the + New Chat and History menu state fixes; citation parser/planner/repair focused tests passed (`14 unit`, `5` + integration), and the reported comparison prompt returned cited answers in three live agentic + API runs plus one regular API run after the fix; production Compose and VPS override configs + rendered; `kubectl kustomize deploy/k8s`, workflow YAML parsing, `uv pip check`, and + `git diff --check` passed. +- **Note:** raw `bash -n deploy/cron/second-brain-backup` fails on the Windows worktree copy because + of CRLF line endings, but the non-mutating normalized check + `tr -d '\r' < deploy/cron/second-brain-backup | bash -n` passed. + +### 2026-06-05 - Demo loop and eval export closure +- **What:** closed the remaining review findings: duplicate feedback-promotion races now return + `409`, worker-owned searchable jobs invalidate Redis search cache after the final commit, and + durable `eval_cases` rows can be exported as reviewable YAML fragments with + `python -m app.eval.export_cases`. +- **Demo:** added `python -m app.demo.seed` to create the case-study flow end to end: capture-backed + bookmark, cited chat answer, and negative feedback ready for `/feedback` review. +- **Docs:** README, usage guide, implementation notes, and case study now present the demo loop as + capture -> chat/search -> feedback promotion -> eval export/gate. +- **Verified:** focused exporter/demo/worker/promotion regressions passed (`30 passed`); full + backend suite passed (`249 passed, 8 warnings`); eval gate passed at `1.000` for `hit_at_k`, + `citation_validity`, and `refusal_accuracy`; `python -m app.eval.export_cases --help` and + `python -m app.demo.seed --help` loaded cleanly; frontend `npm run lint`, `npm run build`, and + `npm audit --audit-level=high` passed; production Compose, tracked VPS template, and local + gitignored VPS override all rendered with dummy env; `git diff --check` passed with only CRLF + normalization warnings. + +### 2026-06-05 - Runtime default changed to local-first +- **What:** superseded the always-on VPS default with a local-first/on-demand Docker Compose + runtime. The DigitalOcean/Caddy deployment remains documented as an optional cloud demo recipe, + but it is no longer the recommended daily-use path. +- **Docs:** updated AGENTS, README, project plan, Phase 6 plan, usage guide, ADR index, and added + ADR-0015. ADR-0011 is now marked superseded by the local-first runtime decision. +- **Cost/privacy:** default recurring infrastructure cost is now $0; user data stays local by + default except for configured hosted Gemini generation/embedding calls. +- **Operational note:** the current DigitalOcean droplet can be destroyed after local startup is + verified and a fresh droplet Postgres dump/env/backups have been copied and checked locally. + +### 2026-06-05 - Reliability and case-study reassessment pass +- **What:** fixed the two review-blocking reliability issues. Research jobs now let the worker own + the DB transaction during ingest, so a later handler failure rolls back the whole job; reviewed + feedback promotion now writes a durable `eval_cases` Postgres row instead of mutating + `backend/eval/dataset.yaml` from the API. +- **Hardening:** added the `eval_cases` migration/model/RLS coverage, kept promotion validation + against the fixed eval corpus, added CORS preflight coverage for + `X-Second-Brain-Admin-Token`, and added CI Compose rendering for both the production file and the + VPS override template. +- **Docs/demo:** rewrote active privacy/governance wording so retention is described precisely + (raw text is nulled; searchable chunks remain until erasure; hosted Gemini modes send text to + Google) and added `docs/case-study.md` around the tight demo flow: capture -> search/chat -> + feedback promotion/eval gate. +- **Verified:** Alembic upgraded through `0005_eval_cases`; focused regressions passed (`68 + passed`); full backend suite passed (`243 passed, 8 warnings`); eval gate passed at `1.000` for + `hit_at_k`, `citation_validity`, and `refusal_accuracy`; frontend `npm run lint`, `npm run + build`, and `npm audit --audit-level=high` passed; production Compose, tracked VPS template, and + local gitignored VPS override all rendered with dummy env; `git diff --check` passed with only + CRLF normalization warnings. + ### 2026-06-05 - Feedback promotion security findings fixed - **What:** fixed the security review findings on the reviewed feedback-to-eval promotion path. Promotion now requires the normal API bearer plus `X-Second-Brain-Admin-Token`, writes an audit @@ -638,10 +731,9 @@ Add a dated entry per working session. Most recent on top. - **Next:** Phase 0 — design the Postgres schema and produce the ER diagram. ## Open questions / parking lot -- ~~Which VPS provider to buy~~ — RESOLVED 2026-06-02 in ADR-0011: **Oracle Cloud Always Free - (Singapore)** primary ($0, 24 GB, low SEA latency), **Contabo SG ~$5/mo (8 GB)** paid fallback. - Low cost was the priority; refreshed 2026 pricing (Hetzner is EU/US-only → latency cost for a - Vietnam daily-use UI). +- ~~Which VPS provider to buy~~ — SUPERSEDED 2026-06-05 in ADR-0015: default runtime is now + local-first Docker Compose; VPS/cloud hosting is optional and temporary unless explicitly + re-approved. - ~~Chunking strategy specifics (size/overlap)~~ — RESOLVED in ADR-0003 (~512 tok / ~15% overlap, semantic boundaries). - ~~**Install Docker Desktop** before Phase 1 end-to-end / integration tests~~ — DONE 2026-06-01: installed, Phase 0 migration applied live; Docker DB on host **5433** (native PG holds 5432). - ~~Whether to do the optional managed-cluster (GKE/EKS) capstone in Phase 7~~ — DECIDED 2026-06-02 diff --git a/docs/USAGE.md b/docs/USAGE.md index 7428cb7..1694558 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -1,28 +1,25 @@ # Second Brain — Usage Guide -How to use and operate the live deployment. Last verified **2026-06-02** against the -production droplet. Web UI/API surface last updated **2026-06-05**. +How to use and operate the local-first app. The optional VPS deployment was last verified +**2026-06-02** against a DigitalOcean droplet, but ADR-0015 changed the default runtime to +local/on-demand Docker Compose on **2026-06-05**. --- -## Live URLs +## Local URLs -> Replace `YOUR_VPS_IP` in the URLs below with your droplet's public IP address. +Use these for normal daily use: | What | URL | Notes | |---|---|---| -| **Web UI** | **https://YOUR_VPS_IP.sslip.io** | Chat, capture, search, ingest, briefing, feedback, tasks, research, sources, admin. Redirects to `/chat`. | -| **API (app path)** | https://YOUR_VPS_IP.sslip.io/api | Behind Caddy, same TLS cert. e.g. `/api/health`, `/api/chat`. | -| **API (direct)** | http://localhost:8000 *(on the box or via SSH tunnel)* | Plain HTTP, bound to localhost only. Handy for quick `curl`. | -| **Swagger UI** | http://localhost:8000/docs *(on the box or via SSH tunnel)* | Interactive "try it" docs for every endpoint. | -| Metrics | http://localhost:8000/metrics *(on the box or via SSH tunnel)* | Prometheus-format app metrics. Monitoring containers are not started by production Compose. | - -> **TLS:** `YOUR_VPS_IP.sslip.io` is a wildcard-DNS hostname that resolves to the droplet's -> IP (`sslip.io` maps `.sslip.io → `), which lets Caddy obtain a real, auto-renewing -> Let's Encrypt certificate **without owning a domain**. `http://` auto-redirects to `https://`. -> To switch to a real domain later: point an A record at the IP, set `CADDY_SITE_ADDRESS` in -> `deploy/.env.prod` and the frontend build arg in `deploy/docker-compose.vps.yml`, then rebuild -> the frontend + restart Caddy. +| **Web UI** | **http://localhost:3000** | Chat, capture, search, ingest, briefing, feedback, tasks, research, sources, admin. Redirects to `/chat`. | +| **API** | http://localhost:8000 | e.g. `/health`, `/chat`, `/search`. | +| **Swagger UI** | http://localhost:8000/docs | Interactive "try it" docs for every endpoint. | +| Metrics | http://localhost:8000/metrics | Prometheus-format app metrics. | + +For an optional cloud demo, the old Caddy/VPS path is still +`https://YOUR_VPS_IP.sslip.io` with API calls under `/api`. Treat that as temporary unless you +explicitly choose to pay for always-on hosting again. --- @@ -31,21 +28,41 @@ production droplet. Web UI/API surface last updated **2026-06-05**. Second Brain is a personal RAG assistant: you **capture** web passages or **ingest** notes/text, it embeds and stores them in Postgres + pgvector, and you **chat** or **search** over them with **cited** answers. It also -produces a **daily briefing** and exposes **agentic tools** over MCP. The LLM -(`gemini-2.5-flash`) and embeddings (`gemini-embedding-001`) are hosted Gemini API calls, so the -box needs no GPU and fits in 2 GB RAM. +produces a **briefing** and exposes **agentic tools** over MCP. The LLM (`gemini-2.5-flash`) +can be a hosted Gemini API call, while embeddings can be local MiniLM or hosted Gemini depending +on your privacy/performance setting. -**Architecture:** one Docker Compose project (`second-brain`) on one DigitalOcean droplet: -`caddy` (HTTPS reverse proxy) → `frontend` (Next.js) + `api` (FastAPI); `worker` (daily briefing -and async research); `db` (pgvector) and `redis`. The API exposes Prometheus-format metrics at -`/metrics`; Prometheus/Grafana configs are retained under `deploy/`, but production Compose does not -start monitoring containers until a scanned-clean runtime is selected. +**Architecture:** local-first Docker Compose/processes: `frontend` (Next.js), `api` (FastAPI), +`worker` (briefing and async research), `db` (pgvector), and optional `redis`. The API exposes +Prometheus-format metrics at `/metrics`; Prometheus/Grafana configs are retained under `deploy/` +for optional local/demo monitoring. + +--- + +## Seed the portfolio demo + +From `backend/`, seed the compact case-study loop: + +```bash +python -m app.demo.seed +``` + +The seed command creates a capture-backed bookmark, asks a cited chat question, and records +thumbs-down feedback for review. Open `/feedback`, promote the seeded negative example after +checking the labels, then export staged reviewed cases: + +```bash +python -m app.eval.export_cases --output eval/promoted-cases.yaml +``` + +Review the fragment before copying cases into `eval/dataset.yaml`; the running API never mutates +that source-controlled CI fixture. --- ## Using the Web UI -Open **https://YOUR_VPS_IP.sslip.io**. You get: +Open **http://localhost:3000**. You get: - **/chat** — ask a question; the backend buffers generated chunks until citation/support validation passes, then sends the answer over SSE and finalizes with inline `[1]`,`[2]` citation markers. @@ -53,20 +70,24 @@ Open **https://YOUR_VPS_IP.sslip.io**. You get: past threads (auto-refresh). Thumbs up/down records feedback. A "private mode" toggle routes that turn through the local LLM path instead of Gemini (if configured). If the selected LLM cannot stream, the UI falls back to the non-streaming `/chat` response. + When both `SECOND_BRAIN_AGENTIC_RAG_ENABLED=true` and + `NEXT_PUBLIC_AGENTIC_RAG_ENABLED=true` are set, the composer also shows an agentic RAG toggle. + Agentic turns use non-streaming `/chat`, plan multiple note searches, and show a compact trace in + the answer footer. - **/search** — raw hybrid (vector + full-text) search results with source/tag filters, no LLM. -The browser talks to the API at `…/api` through Caddy (same origin, so no CORS issues). In -production, paste `SECOND_BRAIN_API_TOKEN` into the sidebar key field so chat, conversations, -capture, ingest, search, briefing, feedback, tasks, research, sources, and admin pages include -`Authorization: Bearer ...`. +The browser talks to the local API through `NEXT_PUBLIC_API_BASE_URL` (usually +`http://localhost:8000`). If you set `SECOND_BRAIN_API_TOKEN`, paste the same value into the +sidebar key field so chat, conversations, capture, ingest, search, briefing, feedback, tasks, +research, sources, and admin pages include `Authorization: Bearer ...`. Additional web pages: - **/capture** - save a URL, title, selected text, notes, and tags as a searchable bookmark. - **/ingest** - add manual notes or text documents, with source metadata and tags. - **/briefing** - read the latest stored briefing and recent briefing history. - **/feedback** - review thumbs feedback trends, inspect negative examples, edit eval candidates, - and manually promote reviewed cases into the fixed eval dataset. Promotion requires the admin - token in addition to the normal API bearer. + and manually promote reviewed cases into the durable `eval_cases` table. Promotion requires the + admin token in addition to the normal API bearer. - **/tasks** - create tasks and mark them open, done, or cancelled. - **/research** - enqueue async research jobs with optional public source URLs or pasted source text, then watch queued/running/done/failed status. @@ -77,8 +98,8 @@ Additional web pages: ## Using the API -Base URL `https://YOUR_VPS_IP.sslip.io/api`. Examples use `curl` (works on Windows 11 and -the box). +Base URL for normal use is `http://localhost:8000`. If you temporarily deploy to a VPS, replace +that with `https://YOUR_VPS_IP.sslip.io/api`. Examples use `curl` (works on Windows 11). Production personal-data APIs require the single-owner API bearer token: @@ -96,7 +117,7 @@ stays public for uptime checks. Local development remains keyless unless you set server-side. ```bash -curl -X POST https://YOUR_VPS_IP.sslip.io/api/capture \ +curl -X POST http://localhost:8000/capture \ -H "$API_AUTH" \ -H "Content-Type: application/json" -d '{ "url": "https://example.com/article", @@ -117,7 +138,7 @@ notes returns `status: "duplicate"`. The web page also accepts query-prefill par `bookmark`, `research_note`. Use `manual` for ad-hoc text. ```bash -curl -X POST https://YOUR_VPS_IP.sslip.io/api/ingest \ +curl -X POST http://localhost:8000/ingest \ -H "$API_AUTH" \ -H "Content-Type: application/json" -d '{ "source": {"type": "manual", "name": "My Notes"}, @@ -131,7 +152,7 @@ Re-ingesting identical content is deduped by content hash (`status: "duplicate"` ### Ask — `POST /chat` ```bash -curl -X POST https://YOUR_VPS_IP.sslip.io/api/chat \ +curl -X POST http://localhost:8000/chat \ -H "$API_AUTH" \ -H "Content-Type: application/json" \ -d '{"message": "How should I tune the HNSW index?"}' @@ -141,9 +162,27 @@ Returns `answer` (with `[n]` markers), `citations[]`, token `usage`, `model`, `l `{"message":"…","top_k":8,"filters":{"tags":["postgres"]},"options":{"private_mode":false}}`. If nothing relevant is found it refuses rather than inventing an answer. +Opt-in agentic RAG: + +```bash +SECOND_BRAIN_AGENTIC_RAG_ENABLED=true + +curl -X POST http://localhost:8000/chat \ + -H "$API_AUTH" \ + -H "Content-Type: application/json" \ + -d '{"message": "How should I tune HNSW for recall and latency?", "options": {"agentic": true}}' +``` + +Agentic RAG is read-only in v1. It uses LangGraph to plan 2-4 focused searches over existing +indexed notes, merge/dedupe the evidence, optionally retry weak evidence with the original wording, +and answer through the same citation validator as regular chat. The response includes +`retrieval.agentic` with safe trace metadata: subqueries, hit counts, selected chunk count, +fallback/verifier flags, and the step budget. Planner, verifier, and answer calls all use the same +provider choice as the request, including private mode. + ### Stream an answer - `POST /chat/stream` ```bash -curl -N -X POST https://YOUR_VPS_IP.sslip.io/api/chat/stream \ +curl -N -X POST http://localhost:8000/chat/stream \ -H "$API_AUTH" \ -H "Content-Type: application/json" \ -d '{"message": "How should I tune the HNSW index?"}' @@ -161,9 +200,12 @@ If the selected LLM provider cannot stream, the endpoint returns `409` before st should call `/chat` as a fallback. Gemini, Ollama, and the test fake driver currently implement the streaming interface. +Agentic RAG does not stream in v1. `/chat/stream` returns `409` when `options.agentic=true`; clients +should call `/chat` so no answer text is delivered before citation validation. + ### Search — `GET /search` ```bash -curl -H "$API_AUTH" "https://YOUR_VPS_IP.sslip.io/api/search?q=hnsw+tuning&top_k=5" +curl -H "$API_AUTH" "http://localhost:8000/search?q=hnsw+tuning&top_k=5" ``` ### Redis-backed safeguards and caches @@ -214,9 +256,9 @@ Feedback quality endpoints: answer, retrieval, and citation context. - `GET /feedback/eval-candidates` - negative feedback exported as review-first eval candidate cases. `expected_docs` is inferred from cited documents; nothing is promoted automatically. -- `POST /feedback/eval-candidates/{feedback_id}/promote` - append one reviewed case to the fixed - eval dataset. This also requires `X-Second-Brain-Admin-Token`; the body must confirm expected - sources, expected keywords, and refusal behavior. +- `POST /feedback/eval-candidates/{feedback_id}/promote` - persist one reviewed case in Postgres + as a durable `eval_cases` row. This also requires `X-Second-Brain-Admin-Token`; the body must + confirm expected sources, expected keywords, and refusal behavior. Additional API endpoints: - `GET /sources`, `GET /sources/{id}/documents` - source and document overview. @@ -228,9 +270,9 @@ Additional API endpoints: Use feedback analytics to turn thumbs into reviewable quality data: ```bash -curl -H "$API_AUTH" "https://YOUR_VPS_IP.sslip.io/api/feedback/analytics?days=30" -curl -H "$API_AUTH" "https://YOUR_VPS_IP.sslip.io/api/feedback/negative?limit=25&days=30" -curl -H "$API_AUTH" "https://YOUR_VPS_IP.sslip.io/api/feedback/eval-candidates?limit=25&days=30" +curl -H "$API_AUTH" "http://localhost:8000/feedback/analytics?days=30" +curl -H "$API_AUTH" "http://localhost:8000/feedback/negative?limit=25&days=30" +curl -H "$API_AUTH" "http://localhost:8000/feedback/eval-candidates?limit=25&days=30" ``` Eval candidate responses mirror the fixed eval dataset shape, but remain review-only: @@ -256,7 +298,7 @@ Eval candidate responses mirror the fixed eval dataset shape, but remain review- Promote exactly one reviewed candidate after editing the labels: ```bash -curl -X POST https://YOUR_VPS_IP.sslip.io/api/feedback/eval-candidates/123/promote \ +curl -X POST http://localhost:8000/feedback/eval-candidates/123/promote \ -H "$API_AUTH" \ -H "X-Second-Brain-Admin-Token: " \ -H "Content-Type: application/json" -d '{ @@ -276,9 +318,17 @@ curl -X POST https://YOUR_VPS_IP.sslip.io/api/feedback/eval-candidates/123/promo Promotion is manual, admin-gated, audited, and validation is strict: promoted cases include a strict `review` block with the feedback id, reviewer identity, timestamp, and confirmation flags. `expected_docs` must be fixed eval corpus document titles, refusal cases must have empty -`expected_docs` and `expected_keywords`, and malformed or duplicate cases return `422` without -changing `backend/eval/dataset.yaml`. CI uses the same loader, so bad reviewed candidates cannot -enter the gate silently. +`expected_docs` and `expected_keywords`, malformed cases return `422`, and duplicate races return +`409` without changing `backend/eval/dataset.yaml`. Promotion stores the reviewed case durably in +Postgres (`dataset_path: "postgres:eval_cases"`). The source-controlled fixed dataset remains the +CI gate artifact, so promoted cases are staged for a deliberate repo change rather than written +from the production container. + +Export staged reviewed cases from `backend/`: + +```bash +python -m app.eval.export_cases --output eval/promoted-cases.yaml +``` ### Source-backed research - `POST /research/jobs` Research does not use a paid search API. Provide your own evidence as public URLs or source text; @@ -287,7 +337,7 @@ in those excerpts, stores a `research_note`, and writes provenance into the stor metadata. ```bash -curl -X POST https://YOUR_VPS_IP.sslip.io/api/research/jobs \ +curl -X POST http://localhost:8000/research/jobs \ -H "$API_AUTH" \ -H "Content-Type: application/json" -d '{ "topic": "reciprocal rank fusion", @@ -309,20 +359,22 @@ research document has the same `sources[]`, `source_count`, and `grounding` fiel --- -## Daily briefing +## Briefing -The `worker` service drains a job queue continuously; a host cron enqueues one `briefing` job a -day. It's already installed at **`/etc/cron.d/second-brain-briefing`** (07:00 server time). Read -it the next morning at `GET /api/briefing`. Each run summarizes documents ingested since the -previous briefing. Enqueue one on demand: +The worker drains a Postgres-backed job queue. In local-first mode, run the worker when you want +briefings or async research jobs processed, and enqueue a briefing manually or with Windows Task +Scheduler while your machine is on. Read the result at `GET /briefing`. Each run summarizes +documents ingested since the previous briefing. + +From `backend/`, with the backend environment loaded: ```bash -cd /root/second-brain -docker compose -p second-brain \ - -f deploy/docker-compose.prod.yml -f deploy/docker-compose.vps.yml \ - --env-file deploy/.env.prod exec -T worker python -m app.jobs.enqueue briefing +python -m app.jobs.enqueue briefing +python -m app.jobs.worker --once ``` +For an optional VPS/cloud demo, use the Compose command in the deploy runbook instead. + --- ## Agentic tools (MCP) @@ -341,7 +393,10 @@ keyless smoke test. --- -## Operating the box +## Optional Cloud Operations + +The commands below are only for a temporary VPS/cloud demo. They are not required for normal +local-first use. ```bash ssh root@YOUR_VPS_IP @@ -377,15 +432,17 @@ is baked into the bundle at build time). |---|---:|---| | `POSTGRES_PASSWORD` | Yes | Database password for the self-hosted Postgres service. | | `SECOND_BRAIN_API_TOKEN` | Yes | Single-owner bearer token for personal-data routes and normal web/API use. | -| `SECOND_BRAIN_ADMIN_TOKEN` | Recommended for data-ops and eval promotion | Enables export, source deletion, retention purge, and fixed-eval promotion when sent as `X-Second-Brain-Admin-Token` alongside the normal API bearer. Leave blank to return 503 from governed endpoints. | +| `SECOND_BRAIN_ADMIN_TOKEN` | Recommended for data-ops and eval promotion | Enables export, source deletion, retention purge, and durable eval-case promotion when sent as `X-Second-Brain-Admin-Token` alongside the normal API bearer. Leave blank to return 503 from governed endpoints. | | `SECOND_BRAIN_GEMINI_API_KEY` | For real Gemini mode | Required when `SECOND_BRAIN_LLM_PROVIDER=gemini`; omit only for `fake` or local Ollama mode. | | `SECOND_BRAIN_MCP_ENABLE_MUTATIONS` | Optional local MCP | Defaults to `false`; set `true` only for trusted local MCP clients that may create tasks or research notes. | -| `NEXT_PUBLIC_API_BASE_URL` | Yes | Browser-visible API base, usually `https://YOUR_VPS_IP.sslip.io/api` in production. | +| `SECOND_BRAIN_AGENTIC_RAG_ENABLED` | Optional RAG experiment | Defaults to `false`; set `true` to allow `/chat` requests with `options.agentic=true`. | +| `NEXT_PUBLIC_API_BASE_URL` | Yes | Browser-visible API base, usually `http://localhost:8000` locally or `https://YOUR_VPS_IP.sslip.io/api` for an optional cloud demo. | +| `NEXT_PUBLIC_AGENTIC_RAG_ENABLED` | Optional web toggle | Defaults to hidden/false; set `true` only when the backend agentic flag is also enabled. | **Health checks:** ```bash curl -fsS localhost:8000/health -curl -fsS https://YOUR_VPS_IP.sslip.io/api/health +curl -fsS http://localhost:8000/health $DC ps $DC exec -T db sh -c 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"' $DC exec -T redis redis-cli ping @@ -422,6 +479,24 @@ Compose no longer includes monitoring containers because the current upstream ve with critical/high CVE findings. Reintroduce monitoring only with scanned-clean images or custom builds. +## Retiring the DigitalOcean droplet + +You can retire the current droplet once all of these are true: + +1. Local startup works: backend health returns OK at `http://localhost:8000/health`, the frontend + opens at `http://localhost:3000`, and you can search/chat against the local database you intend + to keep. +2. You have a fresh Postgres dump from the droplet and have restored or archived it somewhere + local/trusted. +3. Any useful files from `/root/second-brain/deploy/.env.prod`, local uploads, backup archives, + and screenshots/evidence have been copied off the droplet. +4. You no longer need the public `https://YOUR_VPS_IP.sslip.io` demo URL. + +Stopping/powering off a DigitalOcean droplet does **not** end Droplet billing because its compute +resources remain reserved. After the backup is safely copied and verified, destroy the droplet and +remove any unattached volumes, snapshots, reserved IPs, load balancers, or firewall resources you +no longer need. + --- ## Admin / data-ops @@ -429,16 +504,17 @@ builds. `SECOND_BRAIN_ADMIN_TOKEN` enables the governed endpoints. They first pass the normal `SECOND_BRAIN_API_TOKEN` gate, then require the admin token for the destructive/read-all action. Use the normal API bearer plus the separate admin header for these calls: -- `GET /api/data/export?source_id=…` — export a source (GDPR access). -- `DELETE /api/data/sources/{id}` — delete a source + its documents (GDPR erasure). -- `POST /api/admin/retention/purge` — null `raw_text` past the retention TTL. -- `POST /api/feedback/eval-candidates/{feedback_id}/promote` — append a reviewed case to the - fixed eval dataset. +- Local direct route: `GET /data/export?source_id=…` (VPS proxy: `GET /api/data/export?source_id=…`) — export a source (GDPR access). +- Local direct route: `DELETE /data/sources/{id}` (VPS proxy: `DELETE /api/data/sources/{id}`) — delete a source + its documents (GDPR erasure). +- Local direct route: `POST /admin/retention/purge` (VPS proxy: `POST /api/admin/retention/purge`) — null `documents.raw_text` past the retention TTL while + leaving searchable chunks intact. +- Local direct route: `POST /feedback/eval-candidates/{feedback_id}/promote` (VPS proxy: `POST /api/feedback/eval-candidates/{feedback_id}/promote`) — store a reviewed case in the + durable `eval_cases` table. ```bash curl -H "Authorization: Bearer " \ -H "X-Second-Brain-Admin-Token: " \ - "https://YOUR_VPS_IP.sslip.io/api/data/export?source_id=3" + "http://localhost:8000/data/export?source_id=3" ``` --- @@ -480,8 +556,12 @@ services; rotate `POSTGRES_PASSWORD` directly in Postgres and recreate `api`/`wo from source-backed research rather than turning capture into a scraper. 1. **Privacy:** with `SECOND_BRAIN_EMBEDDING_PROVIDER=gemini`, note text is sent to Google at - **ingest** (not just chat). Switch to `local` embeddings for a fully private path (needs a - ≥4 GB box for the torch model). + **ingest** (not just chat). During chat, retrieved chunks and the user question are sent to the + configured generation provider. Switch to `local` embeddings and local Ollama generation for + the private path, with the trade-off that your local machine needs more memory. Retention nulls only the + original `documents.raw_text` copy; `chunks.content` remains searchable until source erasure. + Agentic RAG uses the same provider choice for planning, verification, and answering, but it can + send more search queries and retrieved snippets to that provider during one turn. --- diff --git a/docs/adr/0001-llm-driver-local-vs-hosted.md b/docs/adr/0001-llm-driver-local-vs-hosted.md index 4d0bd08..2015c92 100644 --- a/docs/adr/0001-llm-driver-local-vs-hosted.md +++ b/docs/adr/0001-llm-driver-local-vs-hosted.md @@ -5,6 +5,10 @@ - **Deciders:** project owner - **Context phase:** Phase 0 (formalizes a decision already fixed in the spec) +> **Runtime update 2026-06-05:** ADR-0015 changed the default runtime from a small VPS to +> local-first/on-demand Docker Compose. The Gemini-vs-Ollama abstraction decision still stands; +> the original VPS wording below is historical context. + ## Context The assistant needs an LLM for generation (RAG answers, briefing summaries, research). The runtime diff --git a/docs/adr/0005-hybrid-retrieval-rrf.md b/docs/adr/0005-hybrid-retrieval-rrf.md index b951830..ddc4cf3 100644 --- a/docs/adr/0005-hybrid-retrieval-rrf.md +++ b/docs/adr/0005-hybrid-retrieval-rrf.md @@ -5,6 +5,10 @@ - **Deciders:** project owner - **Context phase:** Phase 1 (retrieval for `/chat`; reused by `/search` in Phase 2) +> **Runtime update 2026-06-05:** ADR-0015 changed the default runtime to local-first Docker +> Compose. The "no second model resident on the VPS" constraint below now means no always-on +> extra model/service in the default local runtime. + ## Context `/chat` must fetch the chunks that ground an answer. The Phase 0 schema carries two diff --git a/docs/adr/0008-evaluation-and-mlflow.md b/docs/adr/0008-evaluation-and-mlflow.md index 759ebc9..d64d7d4 100644 --- a/docs/adr/0008-evaluation-and-mlflow.md +++ b/docs/adr/0008-evaluation-and-mlflow.md @@ -5,6 +5,9 @@ - **Deciders:** project owner (accepted at recommended defaults under the `/goal` directive) - **Context phase:** Phase 3 (Evaluation + MLOps) +> **Runtime update 2026-06-05:** ADR-0015 changes the cost guardrail to "no recurring +> infrastructure bill by default." The local MLflow file-store decision still stands. + ## Context Phase 3 makes answer quality *measurable and improvable* — the JD weights evaluation, A/B, and diff --git a/docs/adr/0011-vps-provider.md b/docs/adr/0011-vps-provider.md index 9a28b0d..2231592 100644 --- a/docs/adr/0011-vps-provider.md +++ b/docs/adr/0011-vps-provider.md @@ -1,6 +1,9 @@ # ADR-0011 — VPS provider for the always-on runtime -- **Status:** Accepted +> **Superseded 2026-06-05 by ADR-0015.** Local-first Docker Compose is now the default runtime; +> this VPS provider decision is retained only as historical context and an optional cloud recipe. + +- **Status:** Superseded by ADR-0015 - **Date:** 2026-06-02 - **Deciders:** project owner (low cost is the stated priority) - **Context phase:** Phase 6 (productionize on a VPS) diff --git a/docs/adr/0012-productionization-and-data-governance.md b/docs/adr/0012-productionization-and-data-governance.md index 3052018..17dee73 100644 --- a/docs/adr/0012-productionization-and-data-governance.md +++ b/docs/adr/0012-productionization-and-data-governance.md @@ -5,13 +5,17 @@ - **Deciders:** project owner (accepted at recommended defaults under the `/goal` directive) - **Context phase:** Phase 6 (productionize + data-ops hardening) +> **Runtime update 2026-06-05:** ADR-0015 changes the default runtime to local-first Docker +> Compose. The governance, CI, metrics, runbook, and optional deploy artifacts in this ADR remain +> accepted; the VPS is no longer required for daily use. + ## Context Phase 6 turns the working app into something operable and governable: data governance (RLS, audit, retention, GDPR access/erasure), connection pooling, observability + alerting, an eval-gated CI/CD pipeline, and ops docs. All of it must be code/config that builds and -tests **without** a purchased VPS (the live deploy is a runbook), stay `$0` in CI (free -minutes, fake LLM), and not break the existing suite. +tests **without** a purchased server, stay `$0` in CI (free minutes, fake LLM), and not break the +existing suite. ## Decision @@ -26,10 +30,12 @@ connects as the table **owner** (bypasses RLS) and the permissive policy covers role (e.g. a future least-privilege app role behind PgBouncer). We do **not** `FORCE` RLS. The policy predicate is the documented seam for real per-tenant scoping if the app goes multi-user. -**Retention nulls `documents.raw_text` after embedding + TTL (D4).** `raw_text` is the only -PII-bearing free text kept post-ingest; `purge_raw_text(older_than_days)` nulls it for embedded -docs past the TTL (`retention_raw_text_days`, default 180). Chunks/embeddings stay — they are -the retrieval units; retention ≠ erasure. Citation rendering already tolerates a purged source. +**Retention nulls `documents.raw_text` after embedding + TTL (D4).** `raw_text` is the original +full-document copy retained for debugging/export while fresh; `purge_raw_text(older_than_days)` +nulls it for embedded docs past the TTL (`retention_raw_text_days`, default 180). Chunks and +embeddings stay because they are the retrieval units, so retention reduces duplicate raw-source +storage but is not anonymization. Erasure is the separate source-delete path. Citation rendering +already tolerates a purged source. **Delete-my-data = source-level erasure with FK cascade (D5).** `delete_source` issues a Core `DELETE` so the DB's `ON DELETE CASCADE` removes documents → chunks → embeddings (the ORM has no @@ -66,7 +72,7 @@ directly (not through the pooler). `docker-compose.prod.yml` is additive and nev `deploy/.env.prod` and `deploy/pgbouncer/userlist.txt` are gitignored. - **Constraint:** admin endpoints are off until a token is set (safe default for a single-user box). RLS is permissive (no second tenant to scope against yet). -- **Deferred:** live VPS deploy (runbook); transaction-mode pooling; remote MLflow; LLM-as-judge +- **Deferred:** optional VPS/cloud deploy; transaction-mode pooling; remote MLflow; LLM-as-judge eval. Frontend currently served via `npm start` in its image (standalone-output optimization left for later given the Next 16 breaking changes). @@ -77,8 +83,8 @@ directly (not through the pooler). `docker-compose.prod.yml` is additive and nev - **FORCE RLS with real row scoping now.** No second user to scope against; forcing it would break owner access and the suite for zero governance gain today. Permissive policy + documented seam is the honest middle. -- **Hard-delete chunks on retention TTL.** Would break search; retention only nulls the raw - source text, erasure (a separate path) removes the subtree. +- **Hard-delete chunks on retention TTL.** Would break search; retention only nulls the original + `documents.raw_text` copy, erasure (a separate path) removes the source subtree. - **Gate CI on answer-text quality.** Needs the real Gemini run (non-deterministic, keyed, costs quota) — unfit for CI; gated metrics are the LLM-independent ones. - **Exercise the full prod stack in CI.** Too heavy; `docker compose config` lint + the deploy diff --git a/docs/adr/0013-briefing-scheduling-and-worker.md b/docs/adr/0013-briefing-scheduling-and-worker.md index c5d1902..f8f0744 100644 --- a/docs/adr/0013-briefing-scheduling-and-worker.md +++ b/docs/adr/0013-briefing-scheduling-and-worker.md @@ -12,8 +12,9 @@ The roadmap's feature is a **morning briefing** you open with coffee, produced b worker drains queued work, a **briefing** summarizes what's new since the last one via the `LLMClient` and is stored for display, and the same worker runs **async `research_topic`** (closing the ADR-0010 Phase-5 deferral). It must reuse every Phase 1 seam, stay testable -without the resident loop, cost `$0`, and run as one more container on the single VPS -(ADR-0011). Phase 5 was skipped when the owner jumped 4 → 6; this picks up the deferred items. +without the resident loop, cost `$0`, and run as a worker process/container wherever the active +runtime is running. Phase 5 was skipped when the owner jumped 4 → 6; this picks up the deferred +items. ADR-0015 later changed the default runtime to local-first/on-demand. ## Decision @@ -23,9 +24,9 @@ without the resident loop, cost `$0`, and run as one more container on the singl for N workers without a `LISTEN/NOTIFY` connection. The `NOTIFY` wake-up (ADR-0004) is deferred as a latency optimization — single-user polling every few seconds is plenty. -**D2 — Scheduler = OS cron enqueues the daily `briefing` job.** The prod stack runs the worker -as a `--loop` service; a host cron line (`python -m app.jobs.enqueue briefing`, in the deploy -runbook) enqueues the briefing daily. `$0`, no resident scheduler dependency. *Rejected:* +**D2 — Scheduler = OS cron enqueues the daily `briefing` job.** The runtime runs the worker +as a `--loop` service/process; a host scheduler line (`python -m app.jobs.enqueue briefing`) +enqueues the briefing when desired. `$0`, no resident scheduler dependency. *Rejected:* APScheduler (extra resident dep), `pg_cron` (extension install). **D3 — Delivery = store-and-display, not email (v1).** Briefings persist to a `briefings` diff --git a/docs/adr/0014-kubernetes-learning-track.md b/docs/adr/0014-kubernetes-learning-track.md index 565a22c..ad7ed6d 100644 --- a/docs/adr/0014-kubernetes-learning-track.md +++ b/docs/adr/0014-kubernetes-learning-track.md @@ -3,15 +3,15 @@ - **Status:** Accepted - **Date:** 2026-06-02 - **Phase:** 7 -- **Supersedes / relates:** ADR-0011 (VPS = production runtime), ADR-0012 (productionization). This - ADR does **not** change the production runtime — it adds a *learning track*. +- **Relates:** ADR-0012 (productionization), ADR-0015 (local-first runtime). This ADR does **not** + change the default runtime — it adds a *learning track*. ## Context -AGENTS.md fixes the production runtime as **one small VPS running Docker Compose** (ADR-0011/0012): +ADR-0015 fixes the default runtime as **local-first Docker Compose**: a single-user app gets no benefit from Kubernetes' multi-node scheduling/autoscaling/self-healing, -and managed K8s would blow the ~$5/mo budget to $70+/mo. But "can operate the app on real -Kubernetes" is a signal worth demonstrating. The roadmap therefore carved out Phase 7 as a +and managed K8s would create a recurring bill. But "can operate the app on real Kubernetes" is a +signal worth demonstrating. The roadmap therefore carved out Phase 7 as a **learning track**: prove the stack runs on real K8s with proper manifests, then tear it down so it costs nothing and is never the prod runtime. The input was the 8-service prod compose stack (`deploy/docker-compose.prod.yml`): `db`, `pgbouncer`, `redis`, `api`, `worker`, `frontend`, diff --git a/docs/adr/0015-local-first-runtime.md b/docs/adr/0015-local-first-runtime.md new file mode 100644 index 0000000..a1145ad --- /dev/null +++ b/docs/adr/0015-local-first-runtime.md @@ -0,0 +1,63 @@ +# ADR-0015 — Local-first runtime; VPS is optional + +- **Status:** Accepted +- **Date:** 2026-06-05 +- **Deciders:** project owner +- **Context phase:** Post-roadmap runtime reassessment +- **Supersedes:** ADR-0011 as the default runtime/provider decision +- **Relates:** ADR-0012 productionization + data governance, ADR-0014 Kubernetes learning track + +## Context + +The project originally optimized for a small always-on VPS so the assistant could be available +24/7. In practice, the owner uses the app intermittently. Paying a DigitalOcean droplet to sit +idle conflicts with the cost-conscious goal, even if the monthly amount is small. + +The app already has the important engineering artifacts without requiring always-on hosting: +Docker Compose, FastAPI, Postgres + pgvector, Redis, worker jobs, CI eval gates, runbooks, +metrics, RLS, audit, retention, erasure, and Kubernetes learning-track manifests. The VPS path is +therefore useful as a demo or future deployment recipe, but not necessary for daily value. + +## Decision + +Use **local-first Docker Compose** as the default runtime. + +- Run the stack on the owner's machine when needed. +- Stop the stack when not in use. +- Schedule briefings locally only when desired, for example with Windows Task Scheduler or an + explicit enqueue command. +- Keep `deploy/docker-compose.prod.yml`, `deploy/docker-compose.vps.yml.example`, Caddy config, + and VPS runbooks as optional deployment artifacts. +- Treat any VPS, managed database, hosted monitoring service, or managed Kubernetes cluster as an + explicit opt-in cost that requires approval before provisioning. + +For remote access to a locally running instance, prefer a free personal VPN/tunnel such as +Tailscale over a permanent public droplet, unless the owner explicitly wants public HTTPS for a +short demo. + +## Consequences + +- **Good:** default recurring infrastructure cost drops to $0. +- **Good:** the app matches real usage: personal, intermittent, private, and easy to stop. +- **Good:** the portfolio story becomes more honest: the project demonstrates production-grade + controls without pretending a single-user assistant requires 24/7 paid uptime. +- **Good:** user data stays local by default, except for configured hosted Gemini generation or + hosted Gemini embeddings. +- **Trade-off:** no always-on web URL, daily briefing, or remote API unless the local machine is + running or an optional tunnel/cloud deployment is active. +- **Trade-off:** local backups become more important because the database now lives with the local + runtime by default. +- **Retained:** the previous DigitalOcean/Caddy path remains available for temporary demos or a + future always-on reversal. + +## Alternatives considered + +- **Keep the DigitalOcean VPS running.** Simple and already verified, but creates a recurring bill + for low usage. +- **Move to free platform services.** Vercel/Render/Neon/Supabase-style free tiers can work for + demos, but they add sleep/cap behavior, privacy changes, and platform-specific constraints. This + is not simpler than local-first for a personal tool. +- **Oracle Cloud Always Free.** No monthly cost, but capacity/reclamation friction still makes it + less predictable than local/on-demand for this owner. +- **Managed Kubernetes.** Strong demo value, wrong permanent runtime for a single-user app and a + known source of surprise bills. diff --git a/docs/adr/0016-agentic-rag-v1.md b/docs/adr/0016-agentic-rag-v1.md new file mode 100644 index 0000000..2d182ca --- /dev/null +++ b/docs/adr/0016-agentic-rag-v1.md @@ -0,0 +1,70 @@ +# ADR-0016 - Agentic RAG v1 with LangGraph + +- **Status:** Accepted +- **Date:** 2026-06-05 +- **Deciders:** project owner +- **Context phase:** Post-roadmap retrieval quality upgrade + +## Context + +Regular chat already uses hybrid retrieval: pgvector semantic search plus PostgreSQL full-text +search, fused with RRF, followed by a strict citation/support validator. The desired improvement is +not a new datastore or autonomous action system. It is a better retrieval loop for harder questions: +break the question into focused searches, merge the evidence, and answer only from cited notes. + +The project constraints still apply: local-first runtime, no recurring infrastructure cost, no new +privacy surprise, no automatic writes, and eval-gated rollout. + +## Decision + +Add an opt-in **agentic RAG** mode using **LangGraph** as a small request-scoped graph: + +1. `plan_queries` asks the selected `LLMClient` for 2-4 focused search queries. +2. `retrieve_subqueries` runs existing `hybrid_search` for each subquery with the same source/tag + filters as regular chat. +3. `select_context` deduplicates chunks, rewards chunks supported by multiple subqueries, and caps + the final context to `top_k`. +4. `verify_evidence` runs only when no usable evidence was found; it can retry the original user + wording once or refuse. +5. `answer` reuses the normal prompt/citation contract and the same finalization validator as + regular chat. + +V1 is read-only. It does **not** call MCP mutation tools, create tasks, fetch web pages, or write +research notes. It also does not use LangGraph checkpointing; each graph run lives only for the +request. Durable evidence remains the existing `retrievals` rows on the final assistant message. + +The mode is behind two gates: + +- API request option: `options.agentic=true` +- Server setting: `SECOND_BRAIN_AGENTIC_RAG_ENABLED=true` + +The web UI exposes the toggle only when `NEXT_PUBLIC_AGENTIC_RAG_ENABLED=true`. `/chat/stream` +returns `409` for agentic requests; clients should call non-streaming `/chat` so answer text is +only delivered after citation validation. + +Agentic eval configs (`agentic`, `gemini-agentic`) compare this graph against the regular baseline. +The regular RAG path remains the default unless the eval and manual review show a clear quality win. + +## Consequences + +- **Good:** improves recall for multi-part or vague questions without replacing the proven hybrid + search implementation. +- **Good:** preserves the `LLMClient` seam. Gemini, Ollama private mode, and the fake test driver + all run through the same graph. +- **Good:** trace metadata is compact and safe: subqueries, hit counts, selected chunk count, + fallback/verifier flags, and budget. It does not expose private chain-of-thought. +- **Cost:** agentic mode uses extra LLM calls and multiple search queries per turn. It is opt-in and + bounded by `SECOND_BRAIN_AGENTIC_RAG_MAX_SUBQUERIES` and the graph recursion limit. +- **Constraint:** v1 does not token-stream answer deltas. This keeps the existing safety posture: + unvalidated model text is not emitted to the browser. + +## Alternatives considered + +- **Replace regular RAG outright.** Rejected because the baseline is proven, faster, and easier to + reason about. Agentic RAG should earn default status through eval. +- **Build a custom loop without LangGraph.** Rejected because the owner explicitly chose LangGraph + and its graph API gives a clearer future path for human-in-the-loop or async flows. +- **Use LangChain agents or LlamaIndex.** Rejected for v1 because the project already has retrieval, + prompting, eval, and provider seams. A low-level graph adds less architecture drift. +- **Persist graph state.** Deferred. Checkpointing is useful for long-running or interruptible + agents, but chat v1 is synchronous and request-scoped. diff --git a/docs/adr/README.md b/docs/adr/README.md index 2e03799..88c6c3e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -16,7 +16,9 @@ older ones explicitly; we don't edit history. | [0008](0008-evaluation-and-mlflow.md) | Evaluation methodology + MLflow tracking (local file store) | Accepted | | [0009](0009-prompt-versioning-ab-rollback.md) | Prompt versioning, A/B, and rollback (in-code registry + env-var selection) | Accepted | | [0010](0010-mcp-server-and-agentic-actions.md) | MCP server (FastMCP/stdio) + agentic actions: search, tasks, digest, research-this-topic | Accepted | -| [0011](0011-vps-provider.md) | VPS provider: Oracle Always Free (Singapore) primary, Contabo SG paid fallback | Accepted | +| [0011](0011-vps-provider.md) | VPS provider for the former always-on runtime | Superseded by [0015](0015-local-first-runtime.md) | | [0012](0012-productionization-and-data-governance.md) | Productionization + data governance: RLS, audit, retention, GDPR erasure, metrics, eval-gated CI, PgBouncer | Accepted | | [0013](0013-briefing-scheduling-and-worker.md) | Daily briefing: durable poll-based worker, OS-cron scheduling, store-and-display, briefings table | Accepted | | [0014](0014-kubernetes-learning-track.md) | Kubernetes learning track on local kind (manifests + HPA + ingress + CI/CD), then torn down | Accepted | +| [0015](0015-local-first-runtime.md) | Local-first Docker Compose runtime; VPS is optional | Accepted | +| [0016](0016-agentic-rag-v1.md) | Agentic RAG v1 with LangGraph: opt-in read-only plan/subsearch/answer graph | Accepted | diff --git a/docs/case-study.md b/docs/case-study.md new file mode 100644 index 0000000..6974cbd --- /dev/null +++ b/docs/case-study.md @@ -0,0 +1,70 @@ +# Second Brain Case Study + +This is the tight demo story for the project: a personal AI system that captures knowledge, +answers with citations, turns user feedback into reviewed eval cases, and blocks regressions with +an eval gate. + +From `backend/`, seed the loop: + +```bash +python -m app.demo.seed +``` + +## Flow 1: Capture to Searchable Knowledge + +**User action:** save a web passage from `/capture` with a URL, title, selected text, notes, and +tags. + +**System path:** `/capture` validates the URL without scraping it, creates a `bookmark` source, +stores the browser-provided text as a document, chunks it, embeds it, and invalidates hot search +cache entries. + +**Proof to show:** the new source appears in `/sources`, the passage appears in `/search`, and a +chat question over that topic returns a cited answer with clickable source cards. + +## Flow 2: Cited Chat to Quality Review + +**User action:** ask a question in `/chat`, inspect the citations, then submit thumbs-down feedback +when an answer misses the mark. + +**System path:** the backend retrieves hybrid Postgres results, buffers streaming LLM output until +citation/support validation passes, persists the assistant message and retrieval rows, and links the +feedback row to the answer and cited documents. + +**Proof to show:** `/feedback` lists the negative example with the original question, answer, +retrieval context, and cited document titles. The review card pre-fills an eval candidate while +making the reviewer explicitly confirm expected sources, expected keywords, and refusal behavior. + +## Flow 3: Feedback to Eval Gate + +**User action:** promote one reviewed negative-feedback candidate from `/feedback` with the admin +token. + +**System path:** the promotion endpoint validates the reviewed labels against the fixed eval corpus, +stores the case durably in the `eval_cases` Postgres table, and writes an audit row. The production +API does not mutate `backend/eval/dataset.yaml`; source-controlled eval changes remain deliberate +repo changes. + +**Proof to show:** the promotion response reports `dataset_path: "postgres:eval_cases"`, the audit +log records `op: promote_eval_case`, and CI runs `python -m app.eval.gate` against the fixed YAML +dataset. When a staged case should become part of the release gate, export it: + +```bash +python -m app.eval.export_cases --output eval/promoted-cases.yaml +``` + +Then copy the reviewed case into `backend/eval/dataset.yaml` as a normal repo change and let the +gate prove the retrieval and citation floor still holds. + +## Why This Matters + +The project is not just "RAG chat." It demonstrates a product loop: + +1. Capture real knowledge. +2. Retrieve and answer with source-backed citations. +3. Collect quality feedback. +4. Convert feedback into reviewed eval data. +5. Use the eval gate to protect future changes. + +That loop is the portfolio signal: user value, data modeling, retrieval quality, AI safety posture, +and release discipline in one small system. diff --git a/docs/data-model/er-diagram.md b/docs/data-model/er-diagram.md index dcdaa0f..f8fb373 100644 --- a/docs/data-model/er-diagram.md +++ b/docs/data-model/er-diagram.md @@ -1,8 +1,7 @@ # Second Brain — Phase 0 Data Model (ER Diagram) -> **Status: DRAFT — for review.** No migrations are written yet. Review this diagram and the -> [Open design decisions](#open-design-decisions-need-your-sign-off) below; once you sign off I -> turn each real decision into an ADR and write the Alembic migrations. +> **Status:** implemented. The original Phase 0 schema has since grown through Alembic migrations; +> this document captures the relational spine plus the major supporting tables. This is the relational spine for the whole project: the `sources → documents → chunks → embeddings` ingest lineage, the `conversations → messages → retrievals → feedback` chat/evidence lineage, plus @@ -148,6 +147,18 @@ erDiagram timestamptz finished_at "nullable" timestamptz created_at } + + eval_cases { + bigint id PK + text case_id "unique; reviewed eval case label" + bigint feedback_id FK "nullable if source feedback is erased" + text question + jsonb expected_docs + jsonb expected_keywords + boolean expect_refusal + jsonb review "review provenance + confirmations" + timestamptz created_at + } ``` --- @@ -167,6 +178,7 @@ erDiagram | `retrievals` | `btree (message_id)`, `btree (chunk_id)` | citation lookups both directions | | `feedback` | `btree (message_id)` | answer-quality analytics | | `jobs` | `btree (status, scheduled_at)` | worker poll | +| `eval_cases` | `UNIQUE (case_id)`, `btree (feedback_id)` | durable reviewed eval-case storage | --- @@ -200,9 +212,9 @@ Diagram uses `bigint GENERATED ALWAYS AS IDENTITY` (smaller, faster joins/indexe counts. Single-user app → bigint wins. ### D5 — `raw_text` retention -`documents.raw_text` is nullable on purpose: store on ingest, **null it out after embedding** per the -retention/“delete-my-data” story (Phase 6). Chunks keep the text needed for retrieval; the original blob -is disposable. Confirm you want raw text purged post-embed (vs kept for re-chunking convenience). +`documents.raw_text` is nullable on purpose: store on ingest, then null it out after the retention TTL +while preserving chunks for retrieval. This reduces duplicate raw-source storage but is not +anonymization; source erasure is the path that removes documents, chunks, and embeddings. --- diff --git a/docs/implementation-notes.md b/docs/implementation-notes.md index 12bc091..3645fab 100644 --- a/docs/implementation-notes.md +++ b/docs/implementation-notes.md @@ -9,31 +9,90 @@ what I gave up**. Keep it honest — the surprises are the valuable part. --- -## Reviewed feedback promotion into fixed evals (2026-06-05) - -- **What:** added a manual promotion path for thumbs-down feedback. `GET /feedback/eval-candidates` - still exports review-first candidates only; `POST /feedback/eval-candidates/{id}/promote` - requires reviewer confirmation of expected sources, expected keywords, and refusal behavior before - appending the edited case to `backend/eval/dataset.yaml`. A follow-up security review tightened - this write path so promotion also requires `X-Second-Brain-Admin-Token`, stores reviewer - provenance in the eval case itself, records a secondary audit row, and returns only a logical - dataset path to the client. -- **Why:** negative feedback is useful eval material, but only after a human has decided what the - correct evidence, answer keywords, and refusal label should be. The fixed eval loader now enforces - explicit reviewed fields, rejects unknown keys/types, checks expected document titles against the - fixed corpus, and validates the whole dataset before any promoted case is written. -- **Trade-off / what I gave up:** promotion does not copy live user/source snippets into the eval - corpus automatically. If a feedback example depends on a document that is not already in - `backend/eval/corpus`, the reviewer must add a safe synthetic corpus document separately or choose - an existing corpus source; otherwise the promotion endpoint returns `422`. -- **Trade-off / what I gave up:** the API still writes a repo file and a Postgres audit row through - separate durability systems; they cannot be a true ACID transaction. To avoid relying on the DB - audit as the only provenance, promoted eval cases now carry their own strict `review` block with - feedback id, reviewer identity, timestamp, and confirmation flags. The audit row is operational - telemetry, while the dataset remains self-describing if a later DB write is interrupted. -- **Affects:** `backend/app/eval/dataset.py`, `backend/eval/dataset.yaml`, - `backend/app/api/conversations.py`, `backend/app/schemas/feedback.py`, - `frontend/app/feedback/page.tsx`, `frontend/lib/api/{client,types}.ts`, `docs/USAGE.md`. +## Agentic RAG v1 is opt-in, read-only, and request-scoped (2026-06-05) + +- **What:** added a LangGraph-backed `agentic_rag` service for `/chat` requests with + `options.agentic=true` when `SECOND_BRAIN_AGENTIC_RAG_ENABLED=true`. The graph plans bounded + subqueries, runs existing hybrid retrieval for each, merges/dedupes chunks, optionally retries + weak evidence with the original wording, and answers through the same citation finalizer as + regular chat. +- **Why:** the regular RAG path was already strong hybrid retrieval. The improvement needed here is + orchestration over that retriever, not a replacement datastore, paid reranker, or autonomous tool + action layer. +- **Trade-off / what I gave up:** v1 is non-streaming and checkpoint-free. `/chat/stream` returns + `409` for agentic requests so unvalidated answer text is never emitted before citation + validation. LangGraph persistence/human-in-the-loop is deferred until a real async or + approval-based workflow needs it. +- **Trade-off / what I gave up:** agentic planning supersedes the optional single query-rewrite hook + for that request. This avoids multiplying planner plus rewrite LLM calls across every subquery. +- **Reliability follow-up:** citation parsing now accepts grouped markers such as `[1, 2]`, and the + agentic planner parses Markdown-fenced JSON instead of treating fence lines as search queries. +- **Trade-off / what I gave up:** if a generated draft fails citation validation, the backend now + makes one internal repair call asking the same LLM to rewrite the draft so every factual sentence + carries same-sentence citations. This adds latency and one extra provider call only on failed + drafts, but preserves the existing rule that unvalidated text is never persisted or streamed to + the browser. +- **Affects:** `backend/app/agentic_rag/service.py`, `backend/app/api/chat.py`, + `backend/app/eval/{configs,harness,pipeline}.py`, `frontend/app/chat/page.tsx`, + `frontend/components/{ChatComposer,MessageList}.tsx`, `docs/adr/0016-agentic-rag-v1.md`. + +## Demo-loop closure and eval export tooling (2026-06-05) + +- **What:** added an operator export CLI for durable `eval_cases` rows and a seed CLI for the + portfolio loop. `python -m app.eval.export_cases` writes a reviewable YAML fragment; `python -m + app.demo.seed` creates a capture-backed bookmark, cited chat answer, and negative feedback row. +- **Why:** reviewed promotion is intentionally durable in Postgres, but the project still needs a + clean bridge into the source-controlled CI eval dataset when a staged case deserves to become a + release gate. The seed command makes the strongest demo reproducible without hand-entering data. +- **Reliability follow-up:** duplicate eval-case insertion races now return `409` instead of leaking + a DB integrity exception, and worker-owned searchable jobs bump the Redis search-cache epoch after + the final DB commit. +- **Trade-off / what I gave up:** the exporter produces a patch fragment rather than mutating + `eval/dataset.yaml`. That preserves code-review discipline but leaves the final promotion into CI + as an operator action. +- **Affects:** `backend/app/eval/export_cases.py`, `backend/app/demo/seed.py`, + `backend/app/api/conversations.py`, `backend/app/jobs/worker.py`, `README.md`, `docs/USAGE.md`, + `docs/case-study.md`. + +## Local-first runtime replaces always-on VPS default (2026-06-05) + +- **What:** changed the default runtime from an always-on VPS to local-first/on-demand Docker + Compose. The existing DigitalOcean/Caddy/Compose deployment artifacts remain as an optional + cloud demo recipe, and ADR-0015 now supersedes ADR-0011 as the default runtime decision. +- **Why:** the owner uses the app intermittently, so paying a droplet to sit idle violates the + cost-conscious goal even if the monthly cost is small. The project still demonstrates the same + core engineering value locally: RAG, pgvector, eval gates, MCP, jobs, governance, and runbooks. +- **Trade-off / what I gave up:** no permanent public URL, 24/7 briefing job, or always-on remote + API by default. Those return only when the local machine is running, a free personal tunnel is + active, or a deliberately temporary cloud demo is started. +- **Shutdown note:** destroy the current DigitalOcean droplet only after local startup is verified + and a fresh Postgres dump, env file, backup archives, and any useful evidence have been copied + and checked locally. +- **Affects:** `AGENTS.md`, `README.md`, `docs/project-plan.md`, `docs/USAGE.md`, + `docs/phase-6-plan.md`, `docs/adr/0011-vps-provider.md`, + `docs/adr/0015-local-first-runtime.md`, `docs/adr/README.md`, `docs/PROGRESS.md`. + +## Durable reviewed eval-case staging (2026-06-05) + +- **What:** moved reviewed feedback promotion out of direct YAML mutation. `GET + /feedback/eval-candidates` still exports review-first thumbs-down candidates, while `POST + /feedback/eval-candidates/{id}/promote` now validates the reviewed case against the fixed corpus + and stores it as a durable `eval_cases` Postgres row in the same transaction as the audit log. + The response reports the logical dataset path `postgres:eval_cases`. +- **Why:** the previous API path wrote a repo file and then wrote a database audit row, which could + not be made truly atomic. Keeping reviewed cases in Postgres makes promotion durable, + auditable, and rollback-safe without letting the running API edit source-controlled CI fixtures. +- **Trade-off / what I gave up:** promoted feedback no longer changes the fixed CI gate + automatically. When a case deserves to become part of the repo-controlled baseline, an operator + should export/review the `eval_cases` row, add any safe synthetic corpus fixture needed, and commit + the YAML change as a normal code review. +- **Trade-off / what I gave up:** promotion still validates expected document titles against the + fixed corpus, so feedback grounded only in private/live documents remains staged until a safe eval + corpus equivalent exists. +- **Affects:** `backend/app/eval/dataset.py`, `backend/app/db/models.py`, + `backend/migrations/versions/0005_eval_cases.py`, `backend/app/api/conversations.py`, + `backend/tests/integration/test_search.py`, `frontend/app/feedback/page.tsx`, `docs/USAGE.md`, + `docs/case-study.md`. ## CodeRabbit security follow-up posture (2026-06-05) diff --git a/docs/k8s-evidence/11-teardown.txt b/docs/k8s-evidence/11-teardown.txt index 6d9fdfb..73b10d5 100644 --- a/docs/k8s-evidence/11-teardown.txt +++ b/docs/k8s-evidence/11-teardown.txt @@ -1,7 +1,7 @@ Phase 7 evidence — Task 11: teardown (D10 — leave nothing running, $0) ====================================================================== After all evidence was captured, the ephemeral learning-track cluster was deleted. Kubernetes was -NEVER the production runtime (that stays the single-VPS Docker Compose stack). No managed cloud was +NEVER the production runtime (that now stays local-first Docker Compose). No managed cloud was ever created (D9 — OFF by default). Local kind costs $0 whether up or down; this confirms hygiene. $ kind get clusters diff --git a/docs/phase-6-plan.md b/docs/phase-6-plan.md index b051328..af17d7a 100644 --- a/docs/phase-6-plan.md +++ b/docs/phase-6-plan.md @@ -1,18 +1,23 @@ -# Phase 6 — Productionize + data-ops hardening Implementation Plan +# Phase 6 — Operations hardening + optional cloud deploy Implementation Plan +> **Runtime update (2026-06-05):** ADR-0015 supersedes the VPS default. Phase 6 remains complete +> as operations hardening plus an optional cloud deploy recipe, but daily use is now local-first +> Docker Compose so the project does not pay for idle VPS uptime. > Work TDD (red → green → commit), DRY, YAGNI. Pure logic is DB-free unit-tested; services are > integration-tested vs the real Postgres on 5433; deploy/observability artifacts are config, > verified to parse/lint. Commit after each green task. -**Goal:** Turn the working app into something operable: **data governance** (RLS posture, +**Goal:** Turn the working app into something operable without requiring paid always-on hosting: +**data governance** (RLS posture, audit log, retention TTL, GDPR export + delete-my-data), **connection pooling** (PgBouncer), **observability** (Prometheus metrics + Grafana dashboards + alert rules), an **eval-gated CI/CD** pipeline (GitHub Actions blocks merge if answer quality regresses), and the **ops docs** (deploy checklist, backup/restore + incident runbooks, query-tuning before/after). -**Scope reality:** the live VPS deploy is a **documented runbook**, not executed here — the box -is provisioned separately (ADR-0011). Everything in this phase is code/config that builds and -tests **without** a purchased server, mirroring how Phases 1–4 deferred their out-of-scope tails. +**Scope reality:** the default runtime is local-first/on-demand Docker Compose (ADR-0015). +The VPS deploy remains a **documented optional recipe**, not a standing requirement. Everything +in this phase is code/config that builds and tests **without** a purchased server, mirroring how +Phases 1–4 deferred their out-of-scope tails. **Architecture:** keep the Phase 1 seams. New logic lives in **services that take a `db` session** (`app/dataops/*`), tested with the rolled-back `db_session` fixture. Metrics are a @@ -26,13 +31,12 @@ and unit/integration-testable. hardening). New top-level `deploy/` (compose.prod, Dockerfiles, pgbouncer/prometheus/grafana config) and `docs/runbooks/`. New `.github/workflows/ci.yml`. -## Decisions (recommended defaults, revisable — full rationale in ADR-0011 / ADR-0012) +## Decisions (recommended defaults, revisable — full rationale in ADR-0012 / ADR-0015) -- **D1 — VPS = Oracle Cloud Always Free (Singapore) primary, Contabo SG (~$5/mo, 8 GB) paid - fallback.** Low cost is the priority and the app is SEA-local; Oracle is $0 + 24 GB + low - latency, Contabo is the cheap reliable backstop. Hetzner is excellent but EU/US-only (latency). - RAM is the binding constraint (Postgres + torch embedder + monitoring stack), not CPU (the LLM - is offloaded to Gemini). *Recorded in ADR-0011.* +- **D1 — Runtime = local-first Docker Compose; VPS = optional recipe only.** Low cost is the + priority and usage is intermittent, so paying a droplet to idle is the wrong default. Keep the + production Compose/Caddy artifacts as a demo or future always-on recipe, but do not require a + running VPS for daily use. *Recorded in ADR-0015; supersedes ADR-0011 as the default runtime.* - **D2 — Audit log is written app-layer via a service, not DB triggers.** A `record(...)` call in the data-ops paths is portable, unit-testable, and explicit about *actor/action/entity*. *Gave up:* automatic capture of out-of-band SQL — acceptable for a single-user app where all @@ -43,10 +47,11 @@ config) and `docs/runbooks/`. New `.github/workflows/ci.yml`. per-tenant scoping if the app ever goes multi-user. *Gave up:* enforced row scoping now (no second user to scope against). - **D4 — Retention nulls `documents.raw_text` after embedding + TTL; chunks/embeddings stay.** - `raw_text` is the only PII-bearing free text retained post-ingest (the model already flags it - "purged after embedding"); chunks are the retrieval units and must remain. Retrieval/citation - code already tolerates a missing source row (`conversations.py` skips purged chunks). *Gave up:* - hard-deleting chunks on TTL — that would break search; retention ≠ erasure. + `raw_text` is the original full-document copy retained for fresh export/debugging; chunks are the + retrieval units and intentionally remain searchable after the retention purge. Retrieval/citation + code tolerates missing raw text/source material where needed. *Gave up:* hard-deleting chunks on + TTL — that would break search; retention is not anonymization, and source erasure is the path that + removes the document/chunk/embedding subtree. - **D5 — Delete-my-data = delete a `source` (cascades to documents→chunks→embeddings via FK `ON DELETE CASCADE`) + an audit `delete` row; export returns the same subtree as JSON first.** Honest GDPR "right to erasure" + "right to access" at the source granularity (the unit the user @@ -56,10 +61,11 @@ config) and `docs/runbooks/`. New `.github/workflows/ci.yml`. (`hit_at_k`, `citation_validity`, `refusal_accuracy`) which are LLM-independent (D2/ADR-0008). *Gave up:* gating on answer-text quality in CI — that needs the real `gemini` run (manual, documented). The gate is a pure `check_thresholds(...)` (unit-tested) + a `__main__` runner. -- **D7 — `docker-compose.prod.yml` is additive and never run in CI.** It composes db + pgbouncer - + redis + api + frontend + prometheus + grafana for the VPS. Verified with `docker compose - config` (parse/lint). The dev `docker-compose.yml` (db-only) is untouched. *Gave up:* a fully - exercised prod stack in CI — too heavy; `config` lint + the deploy runbook cover it. +- **D7 — `docker-compose.prod.yml` is additive and never run in CI.** It composes db + redis + + api + worker + frontend plus optional Caddy bindings for the single-box recipe. Verified with + `docker compose config` (parse/lint). The dev `docker-compose.yml` (db-only) is untouched. + *Gave up:* a fully exercised optional cloud stack in CI — too heavy; `config` lint + the + runbook cover it. ## File structure (created/modified in this phase) @@ -164,13 +170,13 @@ docs/ full-text candidate queries; capture index-on vs `enable_indexscan=off`/`SET hnsw.ef_search` contrasts; write `docs/query-optimization.md` with the before/after and the takeaway. Commit `docs: query-optimization EXPLAIN ANALYZE before/after`. -12. **ADRs + runbooks + docs (DB-free).** ADR-0011 (VPS), ADR-0012 (productionization + governance); +12. **ADRs + runbooks + docs (DB-free).** ADR-0011 (then VPS), ADR-0012 (productionization + governance); index them. `docs/runbooks/{deploy-checklist,backup-restore,incident-response}.md`. README Phase 6 run/verify. Flip PROGRESS Phase 6 → ✅ with a dated entry; record off-spec calls in implementation-notes. Commit `docs: phase-6 ADRs + runbooks + run/verify + progress`. ## Self-review (against project-plan Phase 6 + JD) -- Deploy Docker Compose to VPS → Task 10 (compose.prod + Dockerfiles) + deploy runbook (Task 12); live deploy deferred ✅ +- Optional Docker Compose cloud recipe → Task 10 (compose.prod + Dockerfiles) + deploy runbook (Task 12); live deploy not required ✅ - CI/CD with eval gate → Tasks 8, 9 ✅ - Prometheus/Grafana + alerting + runbooks → Tasks 6, 10, 12 ✅ - RLS + audit log → Tasks 2, 7 ✅ @@ -179,7 +185,7 @@ docs/ - Backup/restore runbook → Task 12 ✅ - Query-optimization before/after → Task 11 ✅ - Tests alongside code (unit + integration vs real Postgres) → every code task ✅ -- $0 / no recurring bill beyond the one VPS (ADR-0011 picks the $0/cheap box; CI on free minutes) ✅ +- $0 default recurring infrastructure bill (ADR-0015 local-first runtime; CI on free minutes) ✅ ## Known sharp edges (flagged, not placeholders) 1. **RLS must not break the app.** The app connects as the table **owner** (`second_brain`), which diff --git a/docs/phase-7-plan.md b/docs/phase-7-plan.md index b05826c..922a588 100644 --- a/docs/phase-7-plan.md +++ b/docs/phase-7-plan.md @@ -10,7 +10,7 @@ manifests, a Postgres StatefulSet, a migrate Job, Deployments for api/worker/fro **ingress**, **HPA** autoscaling under load, reused **Prometheus + Grafana**, and a **CI/CD** workflow that stands the whole thing up on `kind` and tears it down — then **`kind delete cluster`** so **nothing is left running ($0)**. Kubernetes is a **LEARNING TRACK only**, *not* the production -runtime (that stays the single-VPS Docker Compose stack, ADR-0011/0012). Managed cloud (GKE/EKS) is +runtime (that now stays local-first Docker Compose by ADR-0015). Managed cloud (GKE/EKS) is **off by default** (D9) — no paid resource without an explicit OK. **Scope reality:** this mirrors how Phases 1–6 deferred their out-of-scope tails. The deliverable is diff --git a/docs/project-plan.md b/docs/project-plan.md index 78f6cfe..a5a6201 100644 --- a/docs/project-plan.md +++ b/docs/project-plan.md @@ -6,7 +6,7 @@ ## The product in one line -An always-on personal AI assistant you talk to every day: it ingests your notes, PDFs, bookmarks, and activity into a vector store, answers questions with **cited** RAG, gives you a **morning briefing**, does **semantic search** across everything you've fed it, and can **take actions** (create tasks, send a digest) through tools exposed over **MCP** — running on the **Gemini API free tier** (with a local-model private mode behind the same interface), hosted on **one small VPS (~$4–6/mo)**, so it's cheap, always-on, and yours. +A local-first personal AI assistant you run when you need it: it ingests your notes, PDFs, bookmarks, and activity into a vector store, answers questions with **cited** RAG, gives you a **briefing**, does **semantic search** across everything you've fed it, and can **take actions** (create tasks, send a digest) through tools exposed over **MCP** — running on the **Gemini API free tier** (with a local-model private mode behind the same interface), packaged in Docker Compose so the default recurring infrastructure cost is $0. You are the user. The "tangible value to users" the JD asks for is real on day one because you use it daily. @@ -18,32 +18,32 @@ The role is an **AI Applications Developer**. The screening criteria are LLM int --- -## Tech stack (cost-optimized for 24/7, with résumé-relevant swap points) +## Tech stack (cost-optimized for local/on-demand use, with résumé-relevant swap points) | Layer | What you'll run | Cost | JD-named equivalent (call this out in your README) | |---|---|---|---| | **LLM generation** | **Gemini Flash API** (free tier) as the default driver; **local Ollama** behind the same interface as fallback/private mode | **$0** (~1,500 req/day free) | OpenAI / Anthropic / Cohere — one `LLMClient` interface, swap by config | | **Embeddings** | local `sentence-transformers` (run on ingest only) | **$0** | OpenAI `text-embedding-3`, Gemini embeddings | -| **Vector store** | **pgvector** inside self-hosted Postgres | $0 (on the VM) | Pinecone / Weaviate — same retrieval interface | +| **Vector store** | **pgvector** inside self-hosted Postgres | $0 (local) | Pinecone / Weaviate — same retrieval interface | | **Backend / API** | **Python + FastAPI** | $0 | FastAPI named explicitly in the JD | | **Frontend** | **React + Next.js + TypeScript** | $0 | named as preferred qual | | **Agent tooling** | **MCP server** exposing your tools | $0 | MCP + tool-use / agentic patterns | -| **Primary datastore** | **self-hosted Postgres** (relational + pgvector + full-text + JSONB + analytics) on the VM | $0 (on the VM) | PostgreSQL — used to its full depth, not as a blob store | -| **Cache / hot path** | **Redis** for embedding cache, query cache, rate limiting | $0 (on the VM) | named: Redis | -| **Preprocessing** | Pandas/NumPy for chunking, dedupe, PII scrubbing | $0 | named: Pandas, NumPy | -| **MLOps** | **MLflow** for eval runs + prompt/model versioning | $0 (on the VM) | named: MLflow | -| **Compute (24/7)** | **one small VPS**, everything in Docker Compose | **~$4–6/mo** | the always-on host | -| **Containers / orch.** | Docker Compose = the 24/7 runtime; **K8s manifests proven on local k3s/kind** (Phase 7), not run 24/7 | $0 (local cluster torn down after) | named: Docker, Kubernetes | +| **Primary datastore** | **self-hosted Postgres** (relational + pgvector + full-text + JSONB + analytics) in local Compose | $0 | PostgreSQL — used to its full depth, not as a blob store | +| **Cache / hot path** | **Redis** for embedding cache, query cache, rate limiting | local-first (included) - $0 | named: Redis | +| **Preprocessing** | Chunking, dedupe, URL validation, and source metadata normalization | $0 | named: Pandas, NumPy | +| **MLOps** | **MLflow** for eval runs + prompt/model versioning | $0 local file store | named: MLflow | +| **Compute** | Local/on-demand Docker Compose; optional temporary VPS demo recipe | **$0 default** | operating the app without paying for idle uptime | +| **Containers / orch.** | Docker Compose = default runtime; **K8s manifests proven on local k3s/kind** (Phase 7), not run 24/7 | $0 (local cluster torn down after) | named: Docker, Kubernetes | | **CI/CD** | GitHub Actions (build, test, eval gate, deploy) | $0 (free minutes) | named: GitHub Actions | -| **Observability** | Prometheus + Grafana, self-hosted on the VM | $0 (on the VM) | named: Prometheus, Grafana | +| **Observability** | Prometheus-format metrics plus retained Prometheus/Grafana configs for local or demo runs | $0 | named: Prometheus, Grafana | ### Cost model — what you actually pay -Essentially everything runs in Docker Compose on **one small VPS**, so your only recurring cost is the box itself: roughly **$4–6/month**. Gemini Flash handles inference off-box on its free tier (~1,500 requests/day — plenty for a personal assistant), so there's **no GPU and no per-token bill**. Embeddings run locally and only on ingest, so they're effectively free. Postgres, Redis, MLflow, Prometheus, and Grafana are all self-hosted containers on the same VM — no managed-service fees, no external storage limits. +The default recurring infrastructure cost is **$0** because the app runs locally/on demand. Gemini Flash handles inference off-box on its free tier, so there's **no GPU and no per-token bill** for normal usage. Embeddings can run locally on ingest or through the Gemini embedding API, depending on the privacy/performance trade-off you choose. Postgres, Redis, MLflow, and the worker run in local/dev processes or Compose containers instead of a paid always-on server. -**VPS options (x86, no ARM-capacity lottery, predictable):** Hetzner CX22 (~€4/mo, 2 vCPU / 4 GB), DigitalOcean / Vultr / Linode basic (~$5–6/mo). A 4 GB box comfortably runs the whole stack. **The $0 alternative:** Oracle Cloud Always Free (up to 4 ARM cores / 24 GB RAM, never expires) — far more powerful, but ARM capacity is often hard to provision and idle instances can be reclaimed, so it's the "free but fiddly" path. +**Optional cloud path:** the old VPS/Caddy deployment recipe remains available for a temporary demo or deliberate always-on mode, but it is no longer the default. Any VPS, managed database, paid monitoring, or managed Kubernetes spend must be explicitly approved first. -**Why Gemini Flash API instead of local-only LLM:** keeping a capable model resident 24/7 needs real RAM/CPU (or a GPU) and burns power on a box you're paying for; offloading generation to Gemini's free tier keeps the VPS tiny and cheap. **The privacy trade-off** (note this in your README for the GDPR story): query text and retrieved chunks transit to Google. The plan keeps a **local Ollama path behind the same `LLMClient` interface**, so you can flip to a fully-private, no-external-calls mode — and demonstrating that abstraction is itself a strong engineering signal. +**Why Gemini Flash API instead of local-only LLM:** keeping a capable local model resident needs real RAM/CPU (or a GPU); offloading generation to Gemini's free tier keeps the local stack light. **The privacy trade-off**: query text and retrieved chunks transit to Google. The plan keeps a **local Ollama path behind the same `LLMClient` interface**, so you can flip to a fully-private, no-external-calls mode — and demonstrating that abstraction is itself a strong engineering signal. ### Using your Gemini Ultra subscription (by hand, not via the app's API) @@ -91,8 +91,8 @@ Your Ultra subscription and the Gemini *API* are billed separately — Ultra pow │ Briefing job │ pulls new inputs → summarizes → stores digest └────────────────┘ - All of the above run as containers in ONE Docker Compose stack on a single ~$4–6/mo VPS. - Cross-cutting: Alembic migrations · MLflow (eval + versioning) · Prometheus/Grafana · GitHub Actions CI/CD + All of the above run locally/on demand, with an optional cloud Compose recipe retained for demos. + Cross-cutting: Alembic migrations · MLflow (eval + versioning) · Prometheus metrics · GitHub Actions CI/CD ``` --- @@ -136,9 +136,9 @@ The earlier draft under-used the database. This version makes Postgres a first-c **Pipeline triggers without extra infra.** Postgres `LISTEN/NOTIFY` (or a simple `jobs` table polled by workers) drives ingest and briefing without standing up a separate broker — a deliberate "use the database you already have" choice you can defend in an ADR (and contrast with the Redis/queue alternative). -**Privacy & governance (the GDPR/CCPA story).** Row-level security scopes data access; an `audit_log` table records access/changes; a retention policy (TTL on raw inputs after embedding) and a documented "delete my data" path satisfy the JD's data-governance and anonymization bullets. PII scrubbing happens in preprocessing before storage; the audit + retention tables prove you thought about the lifecycle. +**Privacy & governance (the GDPR/CCPA story).** Row-level security demonstrates the access-control posture; an `audit_log` table records governed actions; a retention policy nulls the original `documents.raw_text` copy after embedding while keeping searchable chunks; and a documented source-level erasure path deletes documents, chunks, embeddings, and related rows. This is not anonymization: searchable chunk text remains until erasure, and hosted Gemini modes send text to Google by design. -**Migrations & operability.** Alembic-versioned migrations (every schema change reviewed and reversible), backward-compatible migration discipline tied into the deploy checklist, connection pooling (PgBouncer) for the always-on service, and a backup/restore runbook. These turn "I use Postgres" into "I operate Postgres." +**Migrations & operability.** Alembic-versioned migrations (every schema change reviewed and reversible), backward-compatible migration discipline tied into runbooks, connection pooling patterns for the optional cloud path, and a backup/restore runbook. These turn "I use Postgres" into "I operate Postgres." **Redis, kept honest.** Redis caches embeddings (don't re-embed identical text), caches hot query results, and enforces rate limits — the things a cache/in-memory store is genuinely better at than Postgres. Keeping both, each for its strength, is the realistic-architecture signal. @@ -151,10 +151,10 @@ The earlier draft under-used the database. This version makes Postgres a first-c | Design/implement AI features (chatbot, search, summarization, recommendations) | Features 1–4: chat, search, briefing, agent | | Robust backend APIs, low latency/high throughput | FastAPI service; latency tracked as an eval metric; Redis caching for embeddings | | Integrate AI with frontend; UI/UX collaboration | Next.js/TS chat UI with streaming responses | -| Data pipelines, preprocessing, quality/privacy/security | Ingest worker: chunking, dedupe, PII scrubbing; Postgres constraints enforce quality; local models = privacy | +| Data pipelines, preprocessing, quality/privacy/security | Ingest path: chunking, dedupe, source metadata, URL validation, and Postgres constraints; local model modes reduce external data sharing | | **PostgreSQL** (named) — modeling, queries, optimization | Normalized relational schema; hybrid pgvector + full-text search; JSONB; materialized views/window functions; HNSW/GIN index tuning with `EXPLAIN ANALYZE` | | Feature stores / structured + unstructured data | Chunks + embeddings + JSONB metadata; relational core ties unstructured text to structured sources/tags | -| Data governance, retention, anonymization (GDPR/CCPA) | Row-level security, `audit_log`, retention TTL, documented delete-my-data path, PII scrubbing before storage | +| Data governance and privacy posture (GDPR/CCPA) | Row-level security, `audit_log`, raw-text retention TTL, source-level export/erasure; no anonymization claim while chunks remain searchable | | Schema versioning / migrations | Alembic versioned, reversible migrations tied to the deploy checklist | | Evaluate models — performance, fairness, bias, latency, reliability | Eval harness in MLflow: answer quality, latency, refusal/bias checks | | A/B testing + rollback strategies | A/B two prompt/model configs; rollback via versioned prompts + feature flag | @@ -162,15 +162,15 @@ The earlier draft under-used the database. This version makes Postgres a first-c | Monitoring: logging, metrics, tracing, alerting, incident response | Prometheus/Grafana dashboards; alerts on latency/error/queue; runbooks | | Clean code, API docs, architecture diagrams, dev guides | OpenAPI docs, this architecture doc, per-service READMEs | | Cross-functional translation of requirements | ADRs + system-design docs (you play PM/DS/Sec) | -| Privacy, security, ethical AI, bias mitigation, explainability | PII scrubbing, GDPR notes, citations = explainability, bias eval | +| Privacy, security, ethical AI, bias mitigation, explainability | Explicit hosted-model privacy trade-off, local/private mode, URL SSRF guards, citations for explainability, refusal eval | | Code reviews, unit/integration tests | Self-review checklist; pytest unit + integration against real Postgres | | **Python proficiency / FastAPI** | Backend core | | **LLM integration (OpenAI/Cohere/Anthropic)** | Abstracted LLM client; swap from Ollama with a config flag | | **Embeddings, vector DBs (Pinecone/Weaviate), retrieval** | pgvector now; documented Pinecone/Weaviate swap | -| **Cloud + DevOps (Docker, CI/CD, K8s)** | Docker Compose runtime; GitHub Actions; K8s track on local k3s/kind (Phase 7); VPS deploy | +| **Cloud + DevOps (Docker, CI/CD, K8s)** | Local-first Docker Compose runtime; GitHub Actions; K8s track on local k3s/kind (Phase 7); optional VPS deploy recipe | | **React/TypeScript frontend** (preferred) | Next.js dashboard | | **MLflow / monitoring (Prometheus/Grafana)** (preferred) | Explicit phases | -| **GDPR/CCPA, anonymization** (preferred) | PII pipeline + privacy README section | +| **GDPR/CCPA privacy controls** (preferred) | Access/export, erasure, retention, audit, and a precise privacy README section; anonymization is not claimed | If you can point at every row of this table in a repo, you are a credible candidate for this role. @@ -190,15 +190,15 @@ If you can point at every row of this table in a repo, you are a credible candid **Phase 5 — Daily briefing + pipelines.** Scheduled summarization job; morning digest. **Shareable:** your actual morning briefing. -**Phase 6 — Productionize + data ops.** Deploy the Docker Compose stack to the VPS; GitHub Actions CI/CD with an **eval gate** (deploy blocked if quality regresses); self-hosted Prometheus/Grafana, alerting, runbooks. Data-layer hardening: RLS + audit log, retention TTL and delete-my-data path, PgBouncer pooling, backup/restore runbook, and a query-optimization pass (`EXPLAIN ANALYZE`, index tuning) with before/after numbers. **Shareable:** a Grafana dashboard + ER diagram + a query-tuning before/after — the "I can model *and* operate this" proof. +**Phase 6 — Operations hardening + data ops.** Make the app operable without requiring paid uptime: GitHub Actions CI/CD with an **eval gate** (deploy blocked if quality regresses), Prometheus-compatible metrics, runbooks, and an optional cloud deploy recipe. Data-layer hardening: RLS + audit log, retention TTL and delete-my-data path, pooling patterns, backup/restore runbook, and a query-optimization pass (`EXPLAIN ANALYZE`, index tuning) with before/after numbers. **Shareable:** metrics/config artifacts + ER diagram + a query-tuning before/after — the "I can model *and* operate this" proof. -**Phase 7 — Kubernetes track (learn it without paying to run it).** *Compose remains the 24/7 runtime — this phase is for K8s competence and the JD bullet, run on a free, ephemeral local cluster and torn down after.* See the dedicated **Kubernetes strategy** section below for the full breakdown. In short: author real manifests, prove them on local **k3s/kind**, demonstrate autoscaling and ingress, and wire CI/CD to the cluster. **Shareable:** a screenshot of your pods scaling under load (`kubectl get hpa`/`get pods`) and your Actions pipeline deploying to the cluster — plus a README note explaining *why* the live system runs on Compose. **Optional capstone (only if you want the shiniest demo):** a short, deliberate run on a managed cluster (GKE/EKS), captured, then deleted — see cost note below. +**Phase 7 — Kubernetes track (learn it without paying to run it).** *Compose remains the default runtime — this phase is for K8s competence and the JD bullet, run on a free, ephemeral local cluster and torn down after.* See the dedicated **Kubernetes strategy** section below for the full breakdown. In short: author real manifests, prove them on local **k3s/kind**, demonstrate autoscaling and ingress, and wire CI/CD to the cluster. **Shareable:** a screenshot of your pods scaling under load (`kubectl get hpa`/`get pods`) and your Actions pipeline deploying to the cluster — plus a README note explaining *why* the real app stays local/on-demand. **Optional capstone (only if you want the shiniest demo):** a short, deliberate run on a managed cluster (GKE/EKS), captured, then deleted — see cost note below. --- ## Kubernetes strategy — learn it, demonstrate it, don't pay to run it 24/7 -**The decision, stated plainly:** the live assistant runs on **Docker Compose** on one small VPS. Kubernetes is deliberately *not* the production runtime, because this is a single-user app — K8s's value (multi-node scheduling, autoscaling under real traffic, self-healing across machines, team rolling-deploys) solves problems you don't have here, and running it 24/7 would mean either a $70+/mo managed cluster or the operational overhead of babysitting a single-node cluster for zero benefit. **Being able to explain that trade-off is a stronger interview signal than "I run everything on K8s"** — it shows you know the tool *and* when not to reach for it. +**The decision, stated plainly:** the real assistant runs on **local-first Docker Compose**. Kubernetes is deliberately *not* the production runtime, because this is a single-user app — K8s's value (multi-node scheduling, autoscaling under real traffic, self-healing across machines, team rolling-deploys) solves problems you don't have here, and running it 24/7 would mean either a managed-cluster bill or the operational overhead of babysitting a cluster for zero benefit. **Being able to explain that trade-off is a stronger interview signal than "I run everything on K8s"** — it shows you know the tool *and* when not to reach for it. **The key insight:** learning K8s does not require K8s *uptime*. Every skill below is practiced on a **free local cluster (k3s or kind)** you spin up, deploy to, screenshot, and tear down — $0, and zero risk to your running app. @@ -209,7 +209,7 @@ If you can point at every row of this table in a repo, you are a credible candid | **Ingress + config** | An ingress controller routing to your services + TLS; templated with **Helm or Kustomize** (dev/prod overlays) | Demonstrates routing and environment config the way real clusters do | | **CI/CD to K8s** | GitHub Actions builds images and `kubectl apply`s to the cluster, eval-gated | The *pipeline* is the artifact and doesn't need a permanent cluster | -**Cost guardrail (important to you):** the entire Phase 7 baseline is **$0** — k3s/kind run on your own machine or even on the same VPS temporarily. **Do not** stand up a managed cluster as a permanent thing. The *only* optional spend is the capstone: deploy to GKE/EKS for a single afternoon to capture a real-cloud rolling-deploy screenshot, then **delete the cluster immediately** (set a calendar reminder; clusters left running are the classic surprise bill). Even that is a few dollars at most — and it's entirely skippable, since the local-cluster proof already backs the JD bullet. +**Cost guardrail (important to you):** the entire Phase 7 baseline is **$0** — k3s/kind run on your own machine and are torn down after evidence capture. **Do not** stand up a managed cluster as a permanent thing. The *only* optional spend is the capstone: deploy to GKE/EKS for a single afternoon to capture a real-cloud rolling-deploy screenshot, then **delete the cluster immediately** (set a calendar reminder; clusters left running are the classic surprise bill). Even that is skippable, since the local-cluster proof already backs the JD bullet. --- @@ -221,7 +221,7 @@ Every workflow from your toolkit still applies, now on an AI system: **ADRs** (l ## What to share, and where -- **LinkedIn:** Phase 2 chat UI, Phase 3 MLflow A/B comparison, Phase 6 architecture diagram + Grafana, Phase 7 K8s autoscaling screenshot (with the "why Compose runs production" note — recruiters love the judgment). Pair each with a short "what I learned" line (RAG, eval, MLOps, MCP, K8s). +- **LinkedIn:** Phase 2 chat UI, Phase 3 MLflow A/B comparison, Phase 6 architecture/metrics/query-tuning writeup, Phase 7 K8s autoscaling screenshot (with the "why the real app stays local-first" note — recruiters love the judgment). Pair each with a short "what I learned" line (RAG, eval, MLOps, MCP, K8s). - **Instagram/Facebook:** the polished chat UI and the morning-briefing screenshot — visual, relatable ("my AI reads my notes for me"). - **Job boards / résumé:** link the repo; lead with the JD-coverage matrix as the README's headline so a recruiter sees the match instantly. diff --git a/frontend/.env.example b/frontend/.env.example index 6aab5b7..10e540b 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -1,2 +1,3 @@ # Copy to frontend/.env.local for local development. NEXT_PUBLIC_API_BASE_URL=http://localhost:8000 +NEXT_PUBLIC_AGENTIC_RAG_ENABLED=false diff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsx index 74ff755..2b41bcf 100644 --- a/frontend/app/chat/page.tsx +++ b/frontend/app/chat/page.tsx @@ -15,16 +15,38 @@ function ChatPage() { const router = useRouter(); const searchParams = useSearchParams(); const cidParam = searchParams.get("cid"); + const routeConversationId = useMemo(() => { + if (!cidParam) return null; + const parsed = Number.parseInt(cidParam, 10); + return Number.isFinite(parsed) ? parsed : null; + }, [cidParam]); const [messages, setMessages] = useState([]); - const [conversationId, setConversationId] = useState( - cidParam ? parseInt(cidParam, 10) : null - ); + const [conversationId, setConversationId] = useState(routeConversationId); const [isSending, setIsSending] = useState(false); const [sourceIds, setSourceIds] = useState([]); const [tags, setTags] = useState([]); + const agenticAvailable = process.env.NEXT_PUBLIC_AGENTIC_RAG_ENABLED === "true"; const bottomRef = useRef(null); const abortRef = useRef(null); + const routeConversationIdRef = useRef(routeConversationId); + const preserveMessagesForRouteIdRef = useRef(null); + + useEffect(() => { + if (routeConversationIdRef.current === routeConversationId) return; + routeConversationIdRef.current = routeConversationId; + + const shouldPreserveMessages = + routeConversationId != null && preserveMessagesForRouteIdRef.current === routeConversationId; + preserveMessagesForRouteIdRef.current = null; + + if (!shouldPreserveMessages) { + abortRef.current?.abort(); + setIsSending(false); + setMessages([]); + } + setConversationId(routeConversationId); + }, [routeConversationId]); const { data: history } = useQuery({ queryKey: ["conversation", conversationId], @@ -33,7 +55,7 @@ function ChatPage() { }); const historyMessages = useMemo(() => { - if (!history) return []; + if (!history || history.id !== conversationId) return []; return history.messages.map((m) => { if (m.role === "assistant") { // Rehydrate the live-chat shape so replayed history gets clickable [n] @@ -60,7 +82,7 @@ function ChatPage() { } return { role: "user" as const, content: m.content }; }); - }, [history]); + }, [history, conversationId]); const displayMessages = messages.length > 0 ? messages : historyMessages; const hasStreamingMessage = displayMessages.some((m) => m.isStreaming); @@ -76,6 +98,7 @@ function ChatPage() { return next; }); if (!requestConversationId) { + preserveMessagesForRouteIdRef.current = data.conversation_id; setConversationId(data.conversation_id); router.replace(`/chat?cid=${data.conversation_id}`, { scroll: false }); } @@ -93,7 +116,11 @@ function ChatPage() { }); }; - const sendMessage = async (payload: { message: string; privateMode: boolean }) => { + const sendMessage = async (payload: { + message: string; + privateMode: boolean; + agenticMode: boolean; + }) => { if (isSending) return; const req: ChatRequest = { @@ -103,7 +130,11 @@ function ChatPage() { source_ids: sourceIds.length ? sourceIds : undefined, tags: tags.length ? tags : undefined, }, - options: { private_mode: payload.privateMode, include_chunks: true }, + options: { + private_mode: payload.privateMode, + include_chunks: true, + agentic: payload.agenticMode && agenticAvailable, + }, }; const base = messages.length > 0 ? messages : historyMessages; @@ -112,10 +143,21 @@ function ChatPage() { const controller = new AbortController(); abortRef.current = controller; + const finishIfActive = (data: ChatResponse) => { + if (controller.signal.aborted || abortRef.current !== controller) return; + finishAssistant(data, req.conversation_id ?? null); + }; try { + if (req.options?.agentic) { + const data = await api.chat(req); + finishIfActive(data); + return; + } + await api.chatStream(req, { signal: controller.signal, onDelta: ({ text }) => { + if (controller.signal.aborted || abortRef.current !== controller) return; if (!text) return; setMessages((prev) => { const idx = prev.findLastIndex((m) => m.role === "assistant" && m.isStreaming); @@ -127,23 +169,26 @@ function ChatPage() { return next; }); }, - onComplete: (data) => finishAssistant(data, req.conversation_id ?? null), + onComplete: finishIfActive, }); } catch (err) { if (controller.signal.aborted) return; if (isChatStreamUnavailableError(err)) { try { const data = await api.chat(req); - finishAssistant(data, req.conversation_id ?? null); + finishIfActive(data); } catch (fallbackErr) { + if (controller.signal.aborted || abortRef.current !== controller) return; showAssistantError(fallbackErr); } } else { showAssistantError(err); } } finally { - if (abortRef.current === controller) abortRef.current = null; - setIsSending(false); + if (abortRef.current === controller) { + abortRef.current = null; + if (!controller.signal.aborted) setIsSending(false); + } } }; @@ -164,7 +209,13 @@ function ChatPage() {
- { void sendMessage({ message: msg, privateMode: pm }); }} disabled={isSending} /> + { + void sendMessage({ message: msg, privateMode: pm, agenticMode: am }); + }} + disabled={isSending} + agenticAvailable={agenticAvailable} + />
); } diff --git a/frontend/app/feedback/page.tsx b/frontend/app/feedback/page.tsx index e44a680..ccdacb7 100644 --- a/frontend/app/feedback/page.tsx +++ b/frontend/app/feedback/page.tsx @@ -219,7 +219,7 @@ function EvalCandidateReviewCard({ candidate, adminToken }: { candidate: EvalCan ); }, onSuccess: (response) => { - setPromoted(`${response.case.id} promoted to ${response.dataset_path}`); + setPromoted(`${response.case.id} saved to ${response.dataset_path}`); }, }); diff --git a/frontend/components/ChatComposer.tsx b/frontend/components/ChatComposer.tsx index 518f410..8488d83 100644 --- a/frontend/components/ChatComposer.tsx +++ b/frontend/components/ChatComposer.tsx @@ -2,16 +2,18 @@ import { useState, useRef, useEffect } from "react"; import { motion } from "framer-motion"; -import { PaperPlaneTilt, Lock, LockOpen } from "@phosphor-icons/react"; +import { Brain, PaperPlaneTilt, Lock, LockOpen } from "@phosphor-icons/react"; interface Props { - onSend: (message: string, privateMode: boolean) => void; + onSend: (message: string, privateMode: boolean, agenticMode: boolean) => void; disabled?: boolean; + agenticAvailable?: boolean; } -export function ChatComposer({ onSend, disabled }: Props) { +export function ChatComposer({ onSend, disabled, agenticAvailable = false }: Props) { const [text, setText] = useState(""); const [privateMode, setPrivateMode] = useState(false); + const [agenticMode, setAgenticMode] = useState(false); const [focused, setFocused] = useState(false); const textareaRef = useRef(null); @@ -26,7 +28,7 @@ export function ChatComposer({ onSend, disabled }: Props) { const submit = () => { if (!canSend) return; - onSend(text.trim(), privateMode); + onSend(text.trim(), privateMode, agenticMode && agenticAvailable); setText(""); }; @@ -40,6 +42,14 @@ export function ChatComposer({ onSend, disabled }: Props) {

)} + {agenticMode && agenticAvailable && ( +
+ +

+ Agentic mode - plans multiple note searches before answering +

+
+ )} {/* Composer card */}
+ {agenticAvailable && ( + setAgenticMode((p) => !p)} + className={`flex h-8 w-8 items-center justify-center rounded-xl border transition-all ${ + agenticMode + ? "bg-sky-50 dark:bg-sky-950/40 border-sky-300 dark:border-sky-800 text-sky-600 dark:text-sky-400" + : "border-border text-muted-foreground hover:text-foreground hover:bg-muted" + }`} + aria-label="Toggle agentic RAG" + aria-pressed={agenticMode} + title={agenticMode ? "Agentic RAG ON" : "Agentic RAG OFF"} + > + + + )} + setPrivateMode((p) => !p)} className={`flex h-8 w-8 items-center justify-center rounded-xl border transition-all ${ diff --git a/frontend/components/ConversationSidebar.tsx b/frontend/components/ConversationSidebar.tsx index 24e3049..f6ff18a 100644 --- a/frontend/components/ConversationSidebar.tsx +++ b/frontend/components/ConversationSidebar.tsx @@ -25,7 +25,7 @@ import { } from "@phosphor-icons/react"; import { api, getStoredApiToken, setStoredApiToken } from "@/lib/api/client"; import { queryClient } from "@/lib/query-client"; -import { Suspense, useState, useEffect } from "react"; +import { Suspense, useState, useEffect, useMemo } from "react"; function SidebarContent() { const pathname = usePathname(); @@ -55,6 +55,29 @@ function SidebarContent() { queryFn: () => api.listConversations(), refetchInterval: 15_000, }); + const activeConversationId = activeCid ? Number.parseInt(activeCid, 10) : null; + const historyItems = useMemo(() => { + const groups = new Map["conversations"][number]; + duplicateCount: number; + }>(); + + for (const conversation of data?.conversations ?? []) { + const title = conversation.title?.trim(); + const key = title ? title.toLowerCase() : `id:${conversation.id}`; + const existing = groups.get(key); + if (!existing) { + groups.set(key, { conversation, duplicateCount: 1 }); + continue; + } + existing.duplicateCount += 1; + if (conversation.id === activeConversationId) { + existing.conversation = conversation; + } + } + + return Array.from(groups.values()); + }, [data?.conversations, activeConversationId]); // Theme is unknown during SSR / first client render; reading it before mount // produces an icon/aria-label that differs between server and client and @@ -157,20 +180,20 @@ function SidebarContent() { {/* History */}
- {data && data.conversations.length > 0 && ( + {historyItems.length > 0 && ( <>

Recent

- {data.conversations.map((c, i) => ( + {historyItems.map(({ conversation: c, duplicateCount }, i) => ( - {c.title ?? `Chat ${c.id}`} + {c.title ?? `Chat ${c.id}`} + {duplicateCount > 1 && ( + + x{duplicateCount} + + )} ))} diff --git a/frontend/components/MessageList.tsx b/frontend/components/MessageList.tsx index 59d94f4..c13d94b 100644 --- a/frontend/components/MessageList.tsx +++ b/frontend/components/MessageList.tsx @@ -108,6 +108,11 @@ export function MessageList({ messages, isLoading }: Props) { {msg.response.citations.length} source{msg.response.citations.length > 1 ? "s" : ""} )} + {msg.response.retrieval.agentic && ( + + agentic: {msg.response.retrieval.agentic.subqueries.length} searches / {msg.response.retrieval.agentic.selected_chunks} chunks + + )}
diff --git a/frontend/lib/api/types.ts b/frontend/lib/api/types.ts index 0c1b54c..d41b9bb 100644 --- a/frontend/lib/api/types.ts +++ b/frontend/lib/api/types.ts @@ -9,6 +9,7 @@ export interface ChatFilters { export interface ChatOptions { private_mode?: boolean; include_chunks?: boolean; + agentic?: boolean; } export interface ChatRequest { @@ -41,6 +42,33 @@ export interface Usage { total_tokens: number | null; } +export interface AgenticTrace { + enabled: boolean; + strategy: string; + subqueries: string[]; + subquery_hit_counts: number[]; + deduped_chunks: number; + selected_chunks: number; + weak_evidence: boolean; + planner_failed: boolean; + verifier_used: boolean; + fallback_used: boolean; + step_budget: { + max_subqueries: number; + recursion_limit: number; + }; +} + +export interface ChatRetrieval { + method: string; + candidates_vector?: number; + candidates_vector_raw?: number; + candidates_fulltext?: number; + fused_returned: number; + agentic?: AgenticTrace; + [key: string]: unknown; +} + export interface ChatResponse { conversation_id: number; message_id: number; @@ -49,12 +77,7 @@ export interface ChatResponse { usage: Usage; model: string | null; latency_ms: number; - retrieval: { - method: string; - candidates_vector: number; - candidates_fulltext: number; - fused_returned: number; - }; + retrieval: ChatRetrieval; } export interface ChatStreamDelta {