diff --git a/README.md b/README.md index 0f5cbce..4acd9f7 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,18 @@ Then open: - `http://localhost:3000/feedback` - review the seeded negative feedback candidate. - `http://localhost:3000/status` - confirm DB migration, source counts, worker queue, token state, and model mode. +### Token Setup Note + +Local development can be keyless when `SECOND_BRAIN_API_TOKEN` is unset. If you set +`SECOND_BRAIN_API_TOKEN` in the backend environment, paste that same value into the web UI's +lower-left **API access** field. The frontend saves it in browser local storage and sends it as +`Authorization: Bearer ` for protected app routes such as sources, status, +chat, search, feedback, tasks, research, and admin data reads. + +`SECOND_BRAIN_ADMIN_TOKEN` is separate. Enter it only in the guarded Admin/Feedback action that +needs it, such as source export, source deletion, retention purge, or eval promotion. The Admin +token is intentionally not saved after leaving the page. + When a reviewed candidate should become source-controlled eval data, export staged cases from `backend/`: @@ -306,7 +318,7 @@ Production secrets live in gitignored `deploy/.env.prod`. Required auth variable | Variable | Purpose | |---|---| -| `SECOND_BRAIN_API_TOKEN` | Required by production Compose; bearer token for normal personal-data routes and the web UI sidebar token field. | +| `SECOND_BRAIN_API_TOKEN` | Required by production Compose; bearer token for normal personal-data routes. Paste this same value into the web UI **API access** field. | | `SECOND_BRAIN_ADMIN_TOKEN` | Enables export, source deletion, and retention purge when sent as `X-Second-Brain-Admin-Token` alongside the normal API bearer. | ```bash diff --git a/backend/app/api/sources.py b/backend/app/api/sources.py index e29c497..20e0e11 100644 --- a/backend/app/api/sources.py +++ b/backend/app/api/sources.py @@ -1,21 +1,106 @@ """Source and document overview endpoints.""" from __future__ import annotations +import logging +from datetime import datetime, timezone + from fastapi import APIRouter, Depends, HTTPException, Query, status -from sqlalchemy import distinct, func, select +from sqlalchemy import delete, distinct, func, select from sqlalchemy.orm import Session, selectinload from app import deps -from app.db.models import Chunk, Document, Source +from app.cache.embedding import encode_with_cache +from app.cache.search import bump_search_cache_epoch +from app.dataops import audit +from app.db.models import Chunk, Document, Embedding, Source +from app.ingest.chunking import chunk_text +from app.ingest.hashing import content_hash from app.schemas.sources import ( + DeleteDocumentResponse, + DocumentContentResponse, + DocumentContentUpdateRequest, DocumentListResponse, DocumentSummary, + DocumentUpdateRequest, SourceListResponse, SourceOut, SourceSummary, + SourceUpdateRequest, ) router = APIRouter(dependencies=[Depends(deps.require_api_access)]) +logger = logging.getLogger(__name__) + + +def _chunk_counts_for_documents(db: Session, document_ids: list[int]) -> dict[int, int]: + if not document_ids: + return {} + rows = db.execute( + select(Chunk.document_id, func.count(Chunk.id)) + .where(Chunk.document_id.in_(document_ids)) + .group_by(Chunk.document_id) + ).all() + return {document_id: chunk_count for document_id, chunk_count in rows} + + +def _document_summary(doc: Document, *, chunk_count: int) -> DocumentSummary: + return DocumentSummary( + id=doc.id, + source_id=doc.source_id, + title=doc.title, + external_id=doc.external_id, + content_type=doc.content_type, + content_hash=doc.content_hash, + status=doc.status, + tags=[tag.name for tag in doc.tags], + chunk_count=chunk_count, + raw_text_available=doc.raw_text is not None, + ingested_at=doc.ingested_at, + created_at=doc.created_at, + updated_at=doc.updated_at, + ) + + +def _load_document(db: Session, document_id: int) -> Document | None: + return db.scalars( + select(Document) + .where(Document.id == document_id) + .options( + selectinload(Document.source), + selectinload(Document.tags), + selectinload(Document.chunks), + ) + ).first() + + +def _document_content_response( + doc: Document, + *, + max_chars: int, +) -> DocumentContentResponse: + if doc.raw_text is not None: + content = doc.raw_text + content_source = "raw_text" + elif doc.chunks: + content = "\n\n".join( + chunk.content for chunk in sorted(doc.chunks, key=lambda c: c.chunk_index) + ) + content_source = "chunks" + else: + content = None + content_source = "unavailable" + + truncated = content is not None and len(content) > max_chars + if truncated: + content = content[:max_chars].rstrip() + + return DocumentContentResponse( + source=SourceOut.model_validate(doc.source), + document=_document_summary(doc, chunk_count=len(doc.chunks)), + content=content, + content_source=content_source, + truncated=truncated, + ) @router.get("/sources", response_model=SourceListResponse) @@ -65,6 +150,34 @@ def list_sources( return SourceListResponse(sources=sources, total=total) +@router.patch("/sources/{source_id}", response_model=SourceOut) +def update_source( + source_id: int, + req: SourceUpdateRequest, + db: Session = Depends(deps.get_db), + settings=Depends(deps.get_settings), + _: bool = Depends(deps.require_admin), +): + source = db.get(Source, source_id) + if source is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Source not found") + + previous_name = source.name + source.name = req.name + audit.record( + db, + actor="operator", + action="update", + entity_type="source", + entity_id=source_id, + detail={"field": "name", "previous": previous_name, "next": req.name}, + enabled=settings.audit_enabled, + ) + db.commit() + db.refresh(source) + return SourceOut.model_validate(source) + + @router.get("/sources/{source_id}/documents", response_model=DocumentListResponse) def list_source_documents( source_id: int, @@ -78,32 +191,232 @@ def list_source_documents( docs = db.scalars( select(Document) .where(Document.source_id == source_id) - .options(selectinload(Document.tags), selectinload(Document.chunks)) + .options(selectinload(Document.tags)) .order_by(Document.created_at.desc(), Document.id.desc()) .limit(limit) ).all() + chunk_counts = _chunk_counts_for_documents(db, [doc.id for doc in docs]) total = db.scalar( select(func.count()).select_from(Document).where(Document.source_id == source_id) ) or 0 return DocumentListResponse( source=SourceOut.model_validate(source), documents=[ - DocumentSummary( - id=doc.id, - source_id=doc.source_id, - title=doc.title, - external_id=doc.external_id, - content_type=doc.content_type, - content_hash=doc.content_hash, - status=doc.status, - tags=[tag.name for tag in doc.tags], - chunk_count=len(doc.chunks), - raw_text_available=doc.raw_text is not None, - ingested_at=doc.ingested_at, - created_at=doc.created_at, - updated_at=doc.updated_at, - ) - for doc in docs + _document_summary(doc, chunk_count=chunk_counts.get(doc.id, 0)) for doc in docs ], total=total, ) + + +@router.get("/documents/{document_id}/content", response_model=DocumentContentResponse) +def get_document_content( + document_id: int, + max_chars: int = Query(2000000, ge=500, le=2000000), + db: Session = Depends(deps.get_db), +): + doc = _load_document(db, document_id) + if doc is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + return _document_content_response(doc, max_chars=max_chars) + + +@router.get("/documents/{document_id}/preview", response_model=DocumentContentResponse) +def preview_document( + document_id: int, + max_chars: int = Query(12000, ge=500, le=20000), + db: Session = Depends(deps.get_db), +): + return get_document_content(document_id=document_id, max_chars=max_chars, db=db) + + +@router.patch("/documents/{document_id}", response_model=DocumentSummary) +def update_document( + document_id: int, + req: DocumentUpdateRequest, + db: Session = Depends(deps.get_db), + settings=Depends(deps.get_settings), + _: bool = Depends(deps.require_admin), +): + doc = db.scalars( + select(Document) + .where(Document.id == document_id) + .options(selectinload(Document.tags)) + ).first() + if doc is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + chunk_count = db.scalar( + select(func.count(Chunk.id)).where(Chunk.document_id == document_id) + ) or 0 + previous_title = doc.title + doc.title = req.title + audit.record( + db, + actor="operator", + action="update", + entity_type="document", + entity_id=document_id, + detail={"field": "title", "previous": previous_title, "next": req.title}, + enabled=settings.audit_enabled, + ) + db.commit() + db.refresh(doc) + return _document_summary(doc, chunk_count=chunk_count) + + +@router.patch("/documents/{document_id}/content", response_model=DocumentContentResponse) +def update_document_content( + document_id: int, + req: DocumentContentUpdateRequest, + max_chars: int = Query(2000000, ge=500, le=2000000), + db: Session = Depends(deps.get_db), + settings=Depends(deps.get_settings), + redis_client=Depends(deps.get_redis), + embedder=Depends(deps.get_embedder), + _: bool = Depends(deps.require_admin), +): + doc = _load_document(db, document_id) + if doc is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + next_hash = content_hash(req.content) + duplicate_id = db.scalar( + select(Document.id).where( + Document.source_id == doc.source_id, + Document.content_hash == next_hash, + Document.id != document_id, + ) + ) + if duplicate_id is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"content duplicates document {duplicate_id} in this source", + ) + + pieces = chunk_text( + req.content, + embedder.count_tokens, + settings.chunk_target_tokens, + settings.chunk_overlap_ratio, + ) + vectors = ( + encode_with_cache( + embedder, + [piece.content for piece in pieces], + redis_client=redis_client, + settings=settings, + namespace="content", + ) + if pieces + else [] + ) + if len(vectors) != len(pieces): + logger.error( + "Embedding count mismatch during document content update", + extra={ + "document_id": document_id, + "source_id": doc.source_id, + "piece_count": len(pieces), + "vector_count": len(vectors), + }, + ) + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="embedding service returned an unexpected vector count", + ) + + previous_hash = doc.content_hash + previous_chunk_count = db.scalar( + select(func.count(Chunk.id)).where(Chunk.document_id == document_id) + ) or 0 + + db.execute( + delete(Chunk).where(Chunk.document_id == document_id), + execution_options={"synchronize_session": False}, + ) + doc.raw_text = req.content + doc.content_hash = next_hash + doc.status = "embedded" + doc.ingested_at = datetime.now(timezone.utc) + + for piece, vector in zip(pieces, vectors, strict=True): + chunk = Chunk( + document_id=doc.id, + chunk_index=piece.index, + content=piece.content, + token_count=piece.token_count, + char_start=piece.char_start, + char_end=piece.char_end, + ) + db.add(chunk) + db.flush() + db.add( + Embedding( + chunk_id=chunk.id, + model=embedder.model_name, + dim=embedder.dim, + embedding=vector, + ) + ) + + audit.record( + db, + actor="operator", + action="update", + entity_type="document", + entity_id=document_id, + detail={ + "field": "content", + "previous_hash": previous_hash, + "next_hash": next_hash, + "previous_chunk_count": previous_chunk_count, + "next_chunk_count": len(pieces), + }, + enabled=settings.audit_enabled, + ) + db.commit() + bump_search_cache_epoch(redis_client, settings) + db.expire_all() + updated = _load_document(db, document_id) + if updated is None: # Defensive; the row existed and was not deleted. + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + return _document_content_response(updated, max_chars=max_chars) + + +@router.delete("/documents/{document_id}", response_model=DeleteDocumentResponse) +def delete_document( + document_id: int, + db: Session = Depends(deps.get_db), + settings=Depends(deps.get_settings), + redis_client=Depends(deps.get_redis), + _: bool = Depends(deps.require_admin), +): + doc = db.get(Document, document_id) + if doc is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Document not found") + + source_id = doc.source_id + chunk_count = db.scalar( + select(func.count(Chunk.id)).where(Chunk.document_id == document_id) + ) or 0 + db.execute( + delete(Document).where(Document.id == document_id), + execution_options={"synchronize_session": False}, + ) + audit.record( + db, + actor="operator", + action="delete", + entity_type="document", + entity_id=document_id, + detail={"source_id": source_id, "chunks_deleted": chunk_count}, + enabled=settings.audit_enabled, + ) + db.commit() + bump_search_cache_epoch(redis_client, settings) + return DeleteDocumentResponse( + document_id=document_id, + source_id=source_id, + chunks_deleted=chunk_count, + ) diff --git a/backend/app/schemas/sources.py b/backend/app/schemas/sources.py index 717558c..14737de 100644 --- a/backend/app/schemas/sources.py +++ b/backend/app/schemas/sources.py @@ -3,7 +3,7 @@ from datetime import datetime -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, Field, field_validator class SourceOut(BaseModel): @@ -28,6 +28,12 @@ class SourceListResponse(BaseModel): total: int +class SourceUpdateRequest(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + + name: str = Field(min_length=1, max_length=240) + + class DocumentSummary(BaseModel): id: int source_id: int @@ -48,3 +54,34 @@ class DocumentListResponse(BaseModel): source: SourceOut documents: list[DocumentSummary] total: int + + +class DocumentUpdateRequest(BaseModel): + model_config = ConfigDict(str_strip_whitespace=True) + + title: str = Field(min_length=1, max_length=400) + + +class DocumentContentUpdateRequest(BaseModel): + content: str = Field(min_length=1, max_length=2_000_000) + + @field_validator("content") + @classmethod + def content_must_not_be_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("content must not be blank") + return value + + +class DocumentContentResponse(BaseModel): + source: SourceOut + document: DocumentSummary + content: str | None + content_source: str + truncated: bool + + +class DeleteDocumentResponse(BaseModel): + document_id: int + source_id: int + chunks_deleted: int diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 7996329..78865a9 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -37,4 +37,4 @@ def fake_embedder(): @pytest.fixture def test_settings(): - return Settings(llm_provider="fake", api_token="test-api-token") + return Settings(_env_file=None, llm_provider="fake", api_token="test-api-token") diff --git a/backend/tests/integration/test_sources_api.py b/backend/tests/integration/test_sources_api.py index 2a537c7..ecf57ac 100644 --- a/backend/tests/integration/test_sources_api.py +++ b/backend/tests/integration/test_sources_api.py @@ -1,6 +1,45 @@ """REST API coverage for source/document overview.""" from __future__ import annotations +from contextlib import contextmanager + +from app import deps +from app.config import Settings +from app.db.models import AuditLog, Chunk, Document, Embedding, Source +from app.main import app + +TOKEN = "test-admin-token" +ADMIN = {"X-Second-Brain-Admin-Token": TOKEN} + + +class _ShortVectorEmbedder: + model_name = "short-vector-embedder" + dim = 384 + + def encode(self, texts: list[str]) -> list[list[float]]: + return [[1.0] + [0.0] * 383 for _ in texts[:-1]] + + def count_tokens(self, text: str) -> int: + return len((text or "").split()) + + +@contextmanager +def _enable_admin(): + previous = app.dependency_overrides.get(deps.get_settings) + app.dependency_overrides[deps.get_settings] = lambda: Settings( + _env_file=None, + llm_provider="fake", + api_token="test-api-token", + admin_token=TOKEN, + ) + try: + yield + finally: + if previous is None: + app.dependency_overrides.pop(deps.get_settings, None) + else: + app.dependency_overrides[deps.get_settings] = previous + def _ingest(client, source_name: str) -> int: resp = client.post( @@ -21,6 +60,24 @@ def _ingest(client, source_name: str) -> int: return resp.json()["source_id"] +def _ingest_many(client, source_name: str, documents: list[dict]) -> int: + resp = client.post( + "/ingest", + json={ + "source": {"type": "manual", "name": source_name, "uri": "file://notes.md"}, + "documents": documents, + }, + ) + assert resp.status_code == 200, resp.text + return resp.json()["source_id"] + + +def _first_document_id(client, source_id: int) -> int: + resp = client.get(f"/sources/{source_id}/documents") + assert resp.status_code == 200 + return resp.json()["documents"][0]["id"] + + def test_sources_api_lists_sources_with_counts(client): source_id = _ingest(client, "Source API") @@ -51,3 +108,287 @@ def test_sources_api_lists_documents_for_source(client): def test_sources_api_missing_source_404(client): assert client.get("/sources/99999999/documents").status_code == 404 + + +def test_sources_api_renames_source_and_audits(client, db_session): + with _enable_admin(): + source_id = _ingest(client, "Rename Source") + + resp = client.patch( + f"/sources/{source_id}", + json={"name": "Renamed Source"}, + headers=ADMIN, + ) + + assert resp.status_code == 200, resp.text + assert resp.json()["name"] == "Renamed Source" + assert db_session.get(Source, source_id).name == "Renamed Source" + assert ( + db_session.query(AuditLog) + .filter( + AuditLog.action == "update", + AuditLog.entity_type == "source", + AuditLog.entity_id == source_id, + ) + .count() + == 1 + ) + + +def test_sources_api_rename_source_requires_admin(client): + source_id = _ingest(client, "No Admin Rename") + + resp = client.patch(f"/sources/{source_id}", json={"name": "Rejected"}) + + assert resp.status_code == 503 + + +def test_sources_api_previews_document_content(client): + source_id = _ingest(client, "Content Source") + document_id = _first_document_id(client, source_id) + + resp = client.get(f"/documents/{document_id}/content") + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["source"]["id"] == source_id + assert body["document"]["id"] == document_id + assert body["content_source"] == "raw_text" + assert "source overview test content" in body["content"] + assert body["truncated"] is False + + +def test_sources_api_keeps_preview_route_for_existing_links(client): + source_id = _ingest(client, "Preview Compatibility Source") + document_id = _first_document_id(client, source_id) + + resp = client.get(f"/documents/{document_id}/preview") + + assert resp.status_code == 200, resp.text + assert resp.json()["document"]["id"] == document_id + + +def test_sources_api_previews_document_chunks_when_raw_text_purged(client, db_session): + source_id = _ingest(client, "Chunk Content Source") + document_id = _first_document_id(client, source_id) + doc = db_session.get(Document, document_id) + doc.raw_text = None + db_session.flush() + + resp = client.get(f"/documents/{document_id}/content") + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["content_source"] == "chunks" + assert "source overview test content" in body["content"] + + +def test_sources_api_renames_document_and_audits(client, db_session): + with _enable_admin(): + source_id = _ingest(client, "Document Rename Source") + document_id = _first_document_id(client, source_id) + + resp = client.patch( + f"/documents/{document_id}", + json={"title": "Renamed document"}, + headers=ADMIN, + ) + + assert resp.status_code == 200, resp.text + assert resp.json()["title"] == "Renamed document" + assert db_session.get(Document, document_id).title == "Renamed document" + assert ( + db_session.query(AuditLog) + .filter( + AuditLog.action == "update", + AuditLog.entity_type == "document", + AuditLog.entity_id == document_id, + ) + .count() + == 1 + ) + + +def test_sources_api_updates_document_content_rebuilds_index_and_audits(client, db_session): + with _enable_admin(): + source_id = _ingest(client, "Document Content Edit Source") + document_id = _first_document_id(client, source_id) + old_doc = db_session.get(Document, document_id) + old_hash = old_doc.content_hash + old_chunk_ids = [ + c.id for c in db_session.query(Chunk).filter(Chunk.document_id == document_id) + ] + assert old_chunk_ids + + edited_content = "edited source content with searchable codex marker. " * 60 + resp = client.patch( + f"/documents/{document_id}/content", + json={"content": edited_content}, + headers=ADMIN, + ) + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["content_source"] == "raw_text" + assert body["content"] == edited_content + assert body["document"]["content_hash"] != old_hash + assert body["document"]["chunk_count"] >= 1 + + db_session.expire_all() + doc = db_session.get(Document, document_id) + assert doc.raw_text == edited_content + assert doc.content_hash == body["document"]["content_hash"] + assert doc.status == "embedded" + + new_chunk_ids = [ + c.id for c in db_session.query(Chunk).filter(Chunk.document_id == document_id) + ] + assert new_chunk_ids + assert set(new_chunk_ids).isdisjoint(old_chunk_ids) + assert ( + db_session.query(Embedding).filter(Embedding.chunk_id.in_(old_chunk_ids)).count() + == 0 + ) + assert ( + db_session.query(Embedding).filter(Embedding.chunk_id.in_(new_chunk_ids)).count() + == len(new_chunk_ids) + ) + audit_row = ( + db_session.query(AuditLog) + .filter( + AuditLog.action == "update", + AuditLog.entity_type == "document", + AuditLog.entity_id == document_id, + ) + .one() + ) + assert audit_row.detail["field"] == "content" + assert audit_row.detail["previous_hash"] == old_hash + assert audit_row.detail["next_hash"] == body["document"]["content_hash"] + + +def test_sources_api_update_document_content_preserves_index_on_embedding_mismatch( + client, db_session +): + with _enable_admin(): + source_id = _ingest(client, "Document Content Mismatch Source") + document_id = _first_document_id(client, source_id) + old_doc = db_session.get(Document, document_id) + old_hash = old_doc.content_hash + old_raw_text = old_doc.raw_text + old_chunk_ids = [ + c.id for c in db_session.query(Chunk).filter(Chunk.document_id == document_id) + ] + assert old_chunk_ids + + previous_embedder = app.dependency_overrides.get(deps.get_embedder) + app.dependency_overrides[deps.get_embedder] = lambda: _ShortVectorEmbedder() + try: + resp = client.patch( + f"/documents/{document_id}/content", + json={"content": "new content that cannot be fully embedded " * 60}, + headers=ADMIN, + ) + + assert resp.status_code == 502 + db_session.expire_all() + doc = db_session.get(Document, document_id) + assert doc.content_hash == old_hash + assert doc.raw_text == old_raw_text + new_chunk_ids = [ + c.id for c in db_session.query(Chunk).filter(Chunk.document_id == document_id) + ] + assert new_chunk_ids == old_chunk_ids + finally: + if previous_embedder is None: + app.dependency_overrides.pop(deps.get_embedder, None) + else: + app.dependency_overrides[deps.get_embedder] = previous_embedder + + +def test_sources_api_update_document_content_rejects_duplicate_hash(client): + with _enable_admin(): + source_id = _ingest_many( + client, + "Document Content Duplicate Source", + [ + { + "title": "First", + "content": "first unique source content " * 40, + "content_type": "text/plain", + }, + { + "title": "Second", + "content": "second duplicate target content " * 40, + "content_type": "text/plain", + }, + ], + ) + resp = client.get(f"/sources/{source_id}/documents") + assert resp.status_code == 200 + by_title = {doc["title"]: doc["id"] for doc in resp.json()["documents"]} + + resp = client.patch( + f"/documents/{by_title['First']}/content", + json={"content": "second duplicate target content " * 40}, + headers=ADMIN, + ) + + assert resp.status_code == 409 + assert "duplicates document" in resp.json()["detail"] + + +def test_sources_api_deletes_document_cascade_and_audits(client, db_session): + with _enable_admin(): + source_id = _ingest(client, "Document Delete Source") + document_id = _first_document_id(client, source_id) + chunk_ids = [ + c.id for c in db_session.query(Chunk).filter(Chunk.document_id == document_id) + ] + assert chunk_ids + assert db_session.query(Embedding).filter(Embedding.chunk_id.in_(chunk_ids)).count() >= 1 + + resp = client.delete(f"/documents/{document_id}", headers=ADMIN) + + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["document_id"] == document_id + assert body["source_id"] == source_id + assert body["chunks_deleted"] == len(chunk_ids) + assert db_session.query(Source).filter(Source.id == source_id).count() == 1 + assert db_session.query(Document).filter(Document.id == document_id).count() == 0 + assert db_session.query(Chunk).filter(Chunk.document_id == document_id).count() == 0 + assert db_session.query(Embedding).filter(Embedding.chunk_id.in_(chunk_ids)).count() == 0 + assert ( + db_session.query(AuditLog) + .filter( + AuditLog.action == "delete", + AuditLog.entity_type == "document", + AuditLog.entity_id == document_id, + ) + .count() + == 1 + ) + + +def test_sources_api_document_actions_missing_404(client): + with _enable_admin(): + assert client.get("/documents/99999999/content").status_code == 404 + assert client.get("/documents/99999999/preview").status_code == 404 + assert ( + client.patch( + "/documents/99999999", + json={"title": "Missing"}, + headers=ADMIN, + ).status_code + == 404 + ) + assert ( + client.patch( + "/documents/99999999/content", + json={"content": "Missing content"}, + headers=ADMIN, + ).status_code + == 404 + ) + assert client.delete("/documents/99999999", headers=ADMIN).status_code == 404 diff --git a/backend/tests/unit/test_api_auth.py b/backend/tests/unit/test_api_auth.py index 52bba78..b1f3931 100644 --- a/backend/tests/unit/test_api_auth.py +++ b/backend/tests/unit/test_api_auth.py @@ -62,8 +62,16 @@ def test_require_admin_rejects_api_token_and_accepts_admin_token(): ("get", "/tasks", {}), ("get", "/research/jobs", {}), ("get", "/sources", {}), + ("patch", "/sources/1", {"json": {"name": "renamed"}}), + ("get", "/sources/1/documents", {}), + ("get", "/documents/1/content", {}), + ("get", "/documents/1/preview", {}), + ("patch", "/documents/1", {"json": {"title": "renamed"}}), + ("patch", "/documents/1/content", {"json": {"content": "updated content"}}), + ("delete", "/documents/1", {}), ("get", "/status", {}), ("get", "/data/export", {"params": {"source_id": 1}}), + ("delete", "/data/sources/1", {}), ("post", "/admin/retention/purge", {}), ], ) @@ -86,6 +94,44 @@ def test_personal_data_routes_require_api_token(method: str, path: str, kwargs: app.dependency_overrides.clear() +@pytest.mark.parametrize( + ("method", "path", "kwargs"), + [ + ("patch", "/sources/1", {"json": {"name": "renamed"}}), + ("patch", "/documents/1", {"json": {"title": "renamed"}}), + ("patch", "/documents/1/content", {"json": {"content": "updated content"}}), + ("delete", "/documents/1", {}), + ("get", "/data/export", {"params": {"source_id": 1}}), + ("delete", "/data/sources/1", {}), + ("post", "/admin/retention/purge", {}), + ], +) +def test_admin_only_routes_require_admin_token_after_api_gate( + method: str, path: str, kwargs: dict +): + app.dependency_overrides[deps.get_settings] = lambda: Settings( + _env_file=None, + api_token="api-secret", + admin_token="admin-secret", + metrics_enabled=False, + ) + app.dependency_overrides[deps.get_db] = lambda: object() + app.dependency_overrides[deps.get_embedder] = lambda: object() + app.dependency_overrides[deps.get_redis] = lambda: None + try: + with TestClient(app) as client: + request = getattr(client, method) + response = request( + path, + headers={"Authorization": "Bearer api-secret"}, + **kwargs, + ) + assert response.status_code == 401 + assert response.json()["detail"] == "invalid or missing admin token" + finally: + app.dependency_overrides.clear() + + def test_authenticated_request_passes_api_gate_but_still_validates_request_body(): app.dependency_overrides[deps.get_settings] = lambda: Settings( _env_file=None, diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 7028dc2..822a544 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -23,6 +23,74 @@ Legend: ⬜ not started · 🟡 in progress · ✅ complete Add a dated entry per working session. Most recent on top. +### 2026-06-07 - CodeRabbit review follow-up for admin/source management +- **What:** addressed PR #26 CodeRabbit findings before merge. Document summaries now use SQL + aggregate chunk counts instead of hydrating full chunk bodies, document content updates abort + before mutation when embeddings return a mismatched vector count, document deletion bumps the + search-cache epoch, admin integration tests use scoped settings overrides, and admin-only auth + tests now verify the valid-API-token/missing-admin-token path. +- **Frontend:** made desktop and mobile New chat controls dispatch the same reset event, prevented + duplicate desktop anchor navigation, and removed the redundant manual History API call from + `/chat`. +- **Verified:** focused backend review tests passed (`53 passed`), backend unit suite passed + (`155 passed`), frontend `npm run lint` passed, frontend `npm run build` passed with the existing + multiple-lockfile warning, and `git diff --check` passed. A full local backend run hit existing + shared-test-database residue (`audit_log`/negative feedback rows); PR CI remains the clean + database merge gate. + +### 2026-06-06 - Admin governance console +- **What:** upgraded `/admin` from three standalone forms into a governance/data-safety console + that explains the two-token operating model, shows API/admin/database/corpus guardrail tiles, + pulls source summaries into a source picker, previews the selected source before export/delete, + validates retention purge input, and keeps destructive actions disabled until the admin token and + typed source-id confirmation are present. +- **Frontend:** reused the existing `/status`, `/sources`, `/data/export`, `DELETE /data/sources/{id}`, + and `/admin/retention/purge` contracts; no backend endpoint, migration, or auth-contract change + was added for this pass. +- **Verified:** frontend `npm run lint` passed; frontend `npm run build` passed with the existing + Next.js multiple-lockfile workspace-root warning. Production preview on + `http://localhost:3017/admin` returned `200` and rendered the expected Admin console text. The + browser session hook still failed with the known `command not found: npx` helper issue. +- **Follow-up fix:** changed the Admin source picker from `listSources(500)` to the backend's + validated `listSources(200)` cap, fixing the visible `422` error on `/admin`. +- **Docs follow-up:** clarified in `README.md` that `SECOND_BRAIN_API_TOKEN` is the value to paste + into the web UI's lower-left API access field, while `SECOND_BRAIN_ADMIN_TOKEN` remains a separate + unsaved token for guarded destructive/governance actions. +- **Test follow-up:** made the shared backend `test_settings` fixture ignore local `.env` files, so + auth expectations do not change when a developer has real API/admin tokens configured locally. + +### 2026-06-06 - Sources navigation cleanup +- **What:** cleaned the Operations submenu by removing the standalone Ingest item and making + Sources the active navigation home for both `/sources` and the existing `/ingest` add-source + workflow. +- **Frontend:** added an `Add New Sources` action to the `/sources` header that opens the + existing source-ingest workflow, and renamed that workflow's visible page copy from ingest + language to add-source language while preserving the same API behavior. +- **Verified:** frontend `npm run lint` passed; frontend `npm run build` passed with the existing + Next.js multiple-lockfile workspace-root warning. Production HTTP smoke on + `http://localhost:3017/sources` confirmed `Add New Sources` renders and the sidebar no longer + contains an Ingest submenu link/label; `/ingest` renders the Add New Sources title/copy. + +### 2026-06-06 - Sources page file management +- **What:** upgraded `/sources` from a read-only overview into a source/file management workspace. + Source folders can be renamed or deleted, files can be clicked for file content, renamed, + edited, or deleted, and the page now has a cleaner metrics strip, folder list, file list, + inline confirmations, and file-content panel. +- **Backend:** added admin-guarded `PATCH /sources/{source_id}`, `PATCH /documents/{document_id}`, + `PATCH /documents/{document_id}/content`, and `DELETE /documents/{document_id}` plus read-only + `GET /documents/{document_id}/content`. Document content returns retained raw text when present + and falls back to indexed chunks after retention purges raw text. Content saves rebuild chunks + and embeddings, update the document hash, invalidate search cache, and audit the source/document + changes. +- **Frontend:** added typed API client methods, inline admin-token entry for rename/delete actions, + per-folder and per-file icon actions, document content loading/error/empty states, edit/save/cancel + controls, and guarded delete confirmation by typed id. +- **Verified:** focused backend/API auth tests passed (`45 passed, 1 warning`); frontend + `npm run lint` passed; frontend `npm run build` passed with the existing Next.js multiple-lockfile + workspace-root warning. Browser QA against a production smoke server on `http://localhost:3016/sources` + and updated API on `http://localhost:8011` confirmed real source data, rename/delete controls, + and successful file content loading with no 404, failed fetch, or runtime error. + ### 2026-06-06 - Web UI modernization - **What:** modernized the Second Brain web UI into a quieter local-first command center without changing backend API contracts, auth headers, routes, streaming behavior, citation behavior, or @@ -47,6 +115,9 @@ Add a dated entry per working session. Most recent on top. - **Screenshots:** refreshed `docs/screenshots/ui-home.png`, `ui-chat.png`, `ui-chat-answer.png`, and `ui-status.png` from the production Next server. The committed screenshots use a width below the right-rail breakpoint so local conversation history is not exposed. +- **Follow-up chat reset fix:** changed the sidebar `+ New chat` control to force a fresh `/chat` + navigation and clear chat page state, so it resets correctly from both `/chat?cid=...` and an + already-open `/chat` page. - **Verified:** frontend `npm run lint` passed; frontend `npm run build` passed with the existing Next.js multiple-lockfile workspace-root warning. Full backend suite passed on an isolated test database (`282 passed, 8 warnings`) with fake LLM and Agentic RAG explicitly disabled. Eval gate diff --git a/docs/implementation-notes.md b/docs/implementation-notes.md index d5d422a..92ea028 100644 --- a/docs/implementation-notes.md +++ b/docs/implementation-notes.md @@ -9,6 +9,69 @@ what I gave up**. Keep it honest — the surprises are the valuable part. --- +## Admin console reuses existing ops contracts (2026-06-06) + +- **What:** upgraded `/admin` into a governance/data-safety console using existing `/status` and + `/sources` read models plus the existing export, source deletion, and retention purge mutations. + The page now shows API/admin token state, database/corpus guardrails, a source picker with impact + preview, stricter retention-day validation, and clearer copy around `X-Second-Brain-Admin-Token`. +- **Why:** the Admin page needed to explain itself and reduce operator mistakes without adding + another backend surface during a UI-focused pass. +- **Trade-off / what I gave up:** recent audit-log display and exact retention dry-run counts are + still deferred because the frontend does not currently have a dedicated read endpoint for either. + The page previews source export/delete impact from source summaries and reports the exact purge + count after the guarded action completes. +- **Affects:** `frontend/app/admin/page.tsx`. + +--- + +## Sources file content edits rebuild the index (2026-06-06) + +- **What:** added an admin-guarded document content edit path from the Sources page. The UI labels + the right-side workspace as "File content"; saving replaces `documents.raw_text`, updates the + content hash, deletes old chunks, creates fresh chunks and embeddings, invalidates the search + cache, and writes an audit row without storing raw content in the audit detail. +- **Why:** editing only the displayed text would make the Sources page lie: search, chat citations, + and retention/export behavior all depend on the indexed chunks and document hash. +- **Trade-off / what I gave up:** there is still no document versioning layer. Replacing content + replaces the old chunks, so historical retrieval rows tied to those chunks can be removed by the + existing database cascades. The editor disables saves when the loaded content is truncated to + avoid overwriting a document with a partial slice. +- **Affects:** `backend/app/api/sources.py`, `backend/app/schemas/sources.py`, + `frontend/app/sources/page.tsx`, `frontend/lib/api/{client,types}.ts`. + +--- + +## Sources page mutations stay admin-guarded (2026-06-06) + +- **What:** added source rename, document rename, document content read, and document delete endpoints. + Rename and delete actions require the existing admin token in addition to normal API access; + document content reads remain a read-only API-token route. +- **Why:** source deletion was already treated as a governed data operation, and the new + source/document write actions should not create a weaker mutation path from the Sources page. + Content reads are read-only and follow the same personal-data API guard as search/source listing. +- **Trade-off / what I gave up:** renaming is a little less convenient because the Sources page asks + for the admin token before saving changes. In return, all source/file edits and deletions share a + consistent operator gate and write audit rows. File content uses retained `raw_text` when + available and falls back to indexed chunks when raw text has already been purged. +- **Affects:** `backend/app/api/sources.py`, `backend/app/schemas/sources.py`, + `frontend/app/sources/page.tsx`, `frontend/lib/api/{client,types}.ts`. + +--- + +## New Chat uses full navigation to guarantee reset (2026-06-06) + +- **What:** changed the sidebar `+ New chat` control to dispatch a chat-reset event and then perform + a document navigation to `/chat`. +- **Why:** same-page App Router navigation could clear in-memory messages while preserving the old + `?cid=...` URL, so refreshing reopened the previous conversation. +- **Trade-off / what I gave up:** this button now does a real page navigation instead of a purely + client-side route transition. It is slightly less SPA-like, but it guarantees a fresh chat state + and correct URL from any current chat route. +- **Affects:** `frontend/components/ConversationSidebar.tsx`, `frontend/app/chat/page.tsx`. + +--- + ## Local status panel reports queue-derived worker state (2026-06-06) - **What:** added an authenticated `/status` endpoint and `/status` web page for local runtime diff --git a/frontend/app/admin/page.tsx b/frontend/app/admin/page.tsx index 46ccbba..570281d 100644 --- a/frontend/app/admin/page.tsx +++ b/frontend/app/admin/page.tsx @@ -1,169 +1,537 @@ "use client"; -import { useState } from "react"; -import { useMutation } from "@tanstack/react-query"; -import { DownloadSimple, Shield, Trash, WarningCircle } from "@phosphor-icons/react"; +import type { ReactNode } from "react"; +import { useEffect, useMemo, useState } from "react"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { + ArrowClockwise, + CheckCircle, + Database, + DownloadSimple, + HardDrives, + Key, + Trash, + WarningCircle, + XCircle, +} from "@phosphor-icons/react"; -import { AppButton, AppPage, Field, InlineError, Panel, PanelHeader, StatusPill, TextInput } from "@/components/AppPage"; -import { api } from "@/lib/api/client"; +import { + AppButton, + AppPage, + Field, + InlineError, + LoadingRows, + Panel, + PanelHeader, + SelectControl, + StatusPill, + TextInput, +} from "@/components/AppPage"; +import { api, getStoredApiToken } from "@/lib/api/client"; +import { formatDateTime } from "@/lib/format"; import { queryClient } from "@/lib/query-client"; +type Tone = "neutral" | "success" | "warning" | "danger"; + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error ? error.message : fallback; +} + +function formatNumber(value: number | null | undefined): string { + return new Intl.NumberFormat().format(value ?? 0); +} + +function sourceTypeLabel(type: string): string { + return type.replace(/_/g, " "); +} + +function positiveInteger(value: string): number | undefined { + if (!value.trim()) return undefined; + const numeric = Number(value); + return Number.isInteger(numeric) && numeric > 0 ? numeric : undefined; +} + +function GovernanceTile({ + icon, + label, + value, + detail, + tone = "neutral", +}: { + icon: ReactNode; + label: string; + value: string; + detail: string; + tone?: Tone; +}) { + return ( +
+
+
+ {icon} + {label} +
+ {value} +
+

{detail}

+
+ ); +} + export default function AdminPage() { - const [token, setToken] = useState(""); + const [adminToken, setAdminToken] = useState(""); const [sourceId, setSourceId] = useState(""); const [deleteConfirm, setDeleteConfirm] = useState(""); const [retentionDays, setRetentionDays] = useState("180"); + const [hasApiToken, setHasApiToken] = useState(false); + + useEffect(() => { + const syncToken = () => setHasApiToken(Boolean(getStoredApiToken())); + syncToken(); + window.addEventListener("second-brain-api-token-changed", syncToken); + return () => + window.removeEventListener("second-brain-api-token-changed", syncToken); + }, []); + + const status = useQuery({ + queryKey: ["status"], + queryFn: () => api.getStatus(), + refetchInterval: 15_000, + retry: false, + }); + + const sources = useQuery({ + queryKey: ["sources"], + queryFn: () => api.listSources(200), + retry: false, + }); - const numericSourceId = Number(sourceId); - const hasSource = Number.isInteger(numericSourceId) && numericSourceId > 0; - const hasToken = token.trim().length > 0; + const sourceRows = useMemo( + () => sources.data?.sources ?? [], + [sources.data?.sources], + ); + const selectedSourceId = positiveInteger(sourceId); + const selectedSource = sourceRows.find( + (source) => source.id === selectedSourceId, + ); + const hasAdminToken = adminToken.trim().length > 0; + const retentionDaysValue = positiveInteger(retentionDays); + const retentionDaysValid = !retentionDays.trim() || retentionDaysValue != null; + const deleteConfirmed = + selectedSource != null && deleteConfirm.trim() === String(selectedSource.id); + + const totals = useMemo( + () => + sourceRows.reduce( + (acc, source) => ({ + documents: acc.documents + source.document_count, + chunks: acc.chunks + source.chunk_count, + }), + { documents: 0, chunks: 0 }, + ), + [sourceRows], + ); const exportSource = useMutation({ - mutationFn: () => api.exportSource(numericSourceId, token.trim()), + mutationFn: () => api.exportSource(selectedSource!.id, adminToken.trim()), }); const deleteSource = useMutation({ - mutationFn: () => api.deleteSource(numericSourceId, token.trim()), - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: ["sources"] }); + mutationFn: () => api.deleteSource(selectedSource!.id, adminToken.trim()), + onSuccess: async () => { + setSourceId(""); setDeleteConfirm(""); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["sources"] }), + queryClient.invalidateQueries({ queryKey: ["source-documents"] }), + queryClient.invalidateQueries({ queryKey: ["status"] }), + ]); }, }); const purgeRetention = useMutation({ - mutationFn: () => api.purgeRetention({ - older_than_days: retentionDays.trim() ? Number(retentionDays) : undefined, - adminToken: token.trim(), - }), + mutationFn: () => + api.purgeRetention({ + older_than_days: retentionDaysValue, + adminToken: adminToken.trim(), + }), + onSuccess: async () => { + await queryClient.invalidateQueries({ queryKey: ["status"] }); + }, }); + function refresh() { + void status.refetch(); + void sources.refetch(); + } + + function chooseSource(nextSourceId: string) { + setSourceId(nextSourceId); + setDeleteConfirm(""); + exportSource.reset(); + deleteSource.reset(); + } + + const databaseOk = + status.data?.database.reachable && status.data.database.migrated; + const workerStatus = status.data?.worker.status ?? "unknown"; + return ( + Refresh + + } > -
- - -
- - setToken(event.target.value)} - placeholder="Bearer token" - /> - - - setSourceId(event.target.value)} - className="font-mono" - placeholder="1" - /> - -
- - The token stays in this browser session and is sent only as the additional admin header. -
-
- +
+ + ) : ( + + ) + } + label="API bearer" + value={hasApiToken ? "saved" : "not set"} + detail={ + hasApiToken + ? "Normal personal-data requests include the browser bearer token." + : "Production routes may reject data-ops until the API bearer is saved." + } + tone={hasApiToken ? "success" : "warning"} + /> + } + label="Admin header" + value={hasAdminToken ? "entered" : "empty"} + detail="Sensitive actions send this value only as X-Second-Brain-Admin-Token." + tone={hasAdminToken ? "success" : "warning"} + /> + } + label="Database" + value={databaseOk ? "current" : "check"} + detail={ + databaseOk + ? `Migration ${status.data?.database.migration_current ?? "current"}.` + : status.data?.database.error ?? "Status endpoint has not confirmed migration state." + } + tone={databaseOk ? "success" : "warning"} + /> + } + label="Corpus" + value={`${formatNumber(sourceRows.length)} sources`} + detail={`${formatNumber(totals.documents)} documents, ${formatNumber(totals.chunks)} chunks visible to the admin console.`} + /> +
+ + {(status.error || sources.error) && ( +
+ {status.error && ( + + )} + {sources.error && ( + + )} +
+ )} +
- -
- {exportSource.error && ( - - )} - exportSource.mutate()} - disabled={!hasToken || !hasSource || exportSource.isPending} - variant="secondary" + +
+ - {exportSource.isPending ? "Exporting" : "Export source"} - - {exportSource.data && ( -
-
- {exportSource.data.document_count} documents - {exportSource.data.source.name} + setAdminToken(event.target.value)} + placeholder="SECOND_BRAIN_ADMIN_TOKEN" + /> + + + + chooseSource(event.target.value)} + disabled={sources.isLoading || sourceRows.length === 0} + > + + {sourceRows.map((source) => ( + + ))} + + + + {sources.isLoading && } + + {selectedSource ? ( +
+
+ #{selectedSource.id} + {selectedSource.document_count} docs + {selectedSource.chunk_count} chunks
- 
-                    {JSON.stringify(exportSource.data, null, 2)}
-
+

+ {selectedSource.name} +

+

+ {sourceTypeLabel(selectedSource.type)} / latest document{" "} + {formatDateTime(selectedSource.latest_document_at)} +

+
+ ) : ( +
+ Select a source to preview the exact export/delete target.
)}
- -
- {deleteSource.error && ( - - )} -
- - Deletes the source and cascades through its documents, chunks, and embeddings. + +
+
+ Worker queue + + {workerStatus} + +
+
+ LLM mode + + {status.data?.runtime.llm_provider ?? "unknown"} + +
+
+ Last document + + {formatDateTime(status.data?.knowledge.latest_document_at)} +
- - setDeleteConfirm(event.target.value)} - className="font-mono focus-visible:border-destructive focus-visible:ring-destructive/15" - placeholder={hasSource ? String(numericSourceId) : "Source ID"} - /> - - deleteSource.mutate()} - disabled={!hasToken || !hasSource || deleteConfirm !== String(numericSourceId) || deleteSource.isPending} - variant="dangerSoft" - > - {deleteSource.isPending ? "Deleting" : "Delete source"} - - {deleteSource.data && ( -

- Deleted source #{deleteSource.data.source_id} and {deleteSource.data.documents_deleted} document{deleteSource.data.documents_deleted === 1 ? "" : "s"}. -

- )}
+
+ +
+
+ + +
+ {exportSource.error && ( + + )} + exportSource.mutate()} + disabled={ + !hasAdminToken || !selectedSource || exportSource.isPending + } + variant="secondary" + > + {" "} + {exportSource.isPending ? "Exporting" : "Export source"} + +
+ {selectedSource + ? `${selectedSource.name} will export ${selectedSource.document_count} document${selectedSource.document_count === 1 ? "" : "s"}.` + : "Choose a source to enable export."} +
+
+ + + + +
+ {deleteSource.error && ( + + )} +
+ + + {selectedSource + ? `${selectedSource.name} will remove ${selectedSource.document_count} document${selectedSource.document_count === 1 ? "" : "s"} plus chunks and embeddings.` + : "Choose a source before enabling deletion."} + +
+ + setDeleteConfirm(event.target.value)} + className="font-mono focus-visible:border-destructive focus-visible:ring-destructive/15" + placeholder={ + selectedSource ? String(selectedSource.id) : "Source ID" + } + /> + + deleteSource.mutate()} + disabled={ + !hasAdminToken || + !selectedSource || + !deleteConfirmed || + deleteSource.isPending + } + variant="dangerSoft" + > + {" "} + {deleteSource.isPending ? "Deleting" : "Delete source"} + + {deleteSource.data && ( +

+ Deleted source #{deleteSource.data.source_id} and{" "} + {deleteSource.data.documents_deleted} document + {deleteSource.data.documents_deleted === 1 ? "" : "s"}. +

+ )} +
+ +
- -
- {purgeRetention.error && ( - - )} - - setRetentionDays(event.target.value)} - className="font-mono" - placeholder="180" - /> - - purgeRetention.mutate()} - disabled={!hasToken || purgeRetention.isPending} - variant="secondary" - > - Purge raw text - - {purgeRetention.data && ( -

- Purged {purgeRetention.data.purged} document{purgeRetention.data.purged === 1 ? "" : "s"} older than {purgeRetention.data.older_than_days} days. -

- )} + +
+
+ + { + setRetentionDays(event.target.value); + purgeRetention.reset(); + }} + className="font-mono" + placeholder="180" + /> + + purgeRetention.mutate()} + disabled={ + !hasAdminToken || + !retentionDaysValid || + purgeRetention.isPending + } + variant="secondary" + > + {purgeRetention.isPending ? "Purging" : "Purge raw text"} + +
+
+ {!retentionDaysValid && ( + + )} + {purgeRetention.error && ( + + )} +
+ Scope: all documents with retained raw text older than{" "} + + {retentionDaysValue ?? "the configured TTL"} + {" "} + days. The indexed chunks remain available for search and chat + until source erasure. +
+ {purgeRetention.data && ( +

+ Purged {purgeRetention.data.purged} document + {purgeRetention.data.purged === 1 ? "" : "s"} older than{" "} + {purgeRetention.data.older_than_days} days. +

+ )} +
+ + {exportSource.data && ( + + +
+ 
+                  {JSON.stringify(exportSource.data, null, 2)}
+
+
+ + )}
+ + + +
+
+ Export before erasure when you need a reviewable data snapshot. +
+
+ Use retention purge for privacy cleanup; use source deletion for + full erasure. +
+
+ Do not paste the admin token unless you are actively running one of + these operations. +
+
+ ); } diff --git a/frontend/app/chat/page.tsx b/frontend/app/chat/page.tsx index 8c7d696..4527dfd 100644 --- a/frontend/app/chat/page.tsx +++ b/frontend/app/chat/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState, useRef, useEffect, useMemo, Suspense } from "react"; +import { useState, useRef, useEffect, useMemo, Suspense, useCallback } from "react"; import { useSearchParams, useRouter } from "next/navigation"; import { useQuery } from "@tanstack/react-query"; import { api, isChatStreamUnavailableError } from "@/lib/api/client"; @@ -32,6 +32,25 @@ function ChatPage() { const routeConversationIdRef = useRef(routeConversationId); const preserveMessagesForRouteIdRef = useRef(null); + const resetChatState = useCallback(() => { + abortRef.current?.abort(); + abortRef.current = null; + preserveMessagesForRouteIdRef.current = null; + setIsSending(false); + setMessages([]); + setConversationId(null); + }, []); + + const startNewChat = useCallback(() => { + resetChatState(); + router.replace("/chat", { scroll: false }); + }, [resetChatState, router]); + + useEffect(() => { + window.addEventListener("second-brain-new-chat", startNewChat); + return () => window.removeEventListener("second-brain-new-chat", startNewChat); + }, [startNewChat]); + useEffect(() => { if (routeConversationIdRef.current === routeConversationId) return; routeConversationIdRef.current = routeConversationId; @@ -41,12 +60,10 @@ function ChatPage() { preserveMessagesForRouteIdRef.current = null; if (!shouldPreserveMessages) { - abortRef.current?.abort(); - setIsSending(false); - setMessages([]); + resetChatState(); } setConversationId(routeConversationId); - }, [routeConversationId]); + }, [routeConversationId, resetChatState]); const { data: history } = useQuery({ queryKey: ["conversation", conversationId], diff --git a/frontend/app/ingest/page.tsx b/frontend/app/ingest/page.tsx index 74d67c4..1d529b5 100644 --- a/frontend/app/ingest/page.tsx +++ b/frontend/app/ingest/page.tsx @@ -134,9 +134,9 @@ export default function IngestPage() { return (
@@ -185,12 +185,14 @@ export default function IngestPage() {
{mutation.error && ( - + )} {!mutation.error && !lastResult && (
-

Ready to ingest

+

Ready to add

Results appear here after the API stores and embeds the batch.

@@ -305,7 +307,7 @@ export default function IngestPage() { onClick={() => mutation.mutate()} disabled={!canSubmit} > - {mutation.isPending ? "Ingesting" : "Ingest"} + {mutation.isPending ? "Adding" : "Add source"}
@@ -366,7 +368,7 @@ export default function IngestPage() { onClick={() => mutation.mutate()} disabled={!canSubmit} > - {mutation.isPending ? "Ingesting" : "Ingest"} + {mutation.isPending ? "Adding" : "Add source"}
diff --git a/frontend/app/sources/page.tsx b/frontend/app/sources/page.tsx index a74e82a..fba693b 100644 --- a/frontend/app/sources/page.tsx +++ b/frontend/app/sources/page.tsx @@ -1,22 +1,130 @@ "use client"; -import { useState } from "react"; -import { useQuery } from "@tanstack/react-query"; -import { Books, Database, FileText } from "@phosphor-icons/react"; +import { useMemo, useState } from "react"; +import { useRouter } from "next/navigation"; +import { useMutation, useQuery } from "@tanstack/react-query"; +import { + Books, + Check, + Database, + FilePlus, + FileText, + FolderSimple, + PencilSimple, + Trash, + WarningCircle, + X, +} from "@phosphor-icons/react"; -import { AppPage, EmptyState, InlineError, LoadingRows, Panel, PanelHeader, StatusPill } from "@/components/AppPage"; +import { + AppButton, + AppPage, + EmptyState, + Field, + InlineError, + LoadingRows, + Panel, + PanelHeader, + StatusPill, + TextArea, + TextInput, +} from "@/components/AppPage"; import { api } from "@/lib/api/client"; +import type { DocumentSummary, SourceSummary } from "@/lib/api/types"; import { formatDate, formatDateTime } from "@/lib/format"; +import { queryClient } from "@/lib/query-client"; +import { cn } from "@/lib/utils"; + +type DeleteTarget = + | { kind: "source"; id: number; label: string; documents: number } + | { kind: "document"; id: number; sourceId: number; label: string }; + +function sourceTypeLabel(type: string): string { + return type.replace(/_/g, " "); +} + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error ? error.message : fallback; +} + +function contentSourceLabel(value: string): string { + if (value === "raw_text") return "raw text"; + if (value === "chunks") return "indexed chunks"; + return "unavailable"; +} + +function MetricTile({ + label, + value, + detail, +}: { + label: string; + value: string | number; + detail: string; +}) { + return ( +
+

{label}

+

+ {value} +

+

{detail}

+
+ ); +} + +function DocumentMeta({ doc }: { doc: DocumentSummary }) { + return ( +
+ {doc.chunk_count} chunks + + raw text {doc.raw_text_available ? "kept" : "purged"} + + {doc.tags.map((tag) => ( + + {tag} + + ))} +
+ ); +} export default function SourcesPage() { + const router = useRouter(); const [selectedSourceId, setSelectedSourceId] = useState(null); + const [selectedDocumentId, setSelectedDocumentId] = useState( + null, + ); + const [adminToken, setAdminToken] = useState(""); + const [editingSourceId, setEditingSourceId] = useState(null); + const [sourceNameDraft, setSourceNameDraft] = useState(""); + const [editingDocumentId, setEditingDocumentId] = useState( + null, + ); + const [documentTitleDraft, setDocumentTitleDraft] = useState(""); + const [isEditingContent, setIsEditingContent] = useState(false); + const [contentDraft, setContentDraft] = useState(""); + const [deleteTarget, setDeleteTarget] = useState(null); + const [deleteConfirm, setDeleteConfirm] = useState(""); + + const adminTokenValue = adminToken.trim(); + const hasAdminToken = adminTokenValue.length > 0; const sources = useQuery({ queryKey: ["sources"], queryFn: () => api.listSources(200), }); - const resolvedSourceId = selectedSourceId ?? sources.data?.sources[0]?.id ?? null; + const sourceRows = sources.data?.sources ?? []; + const selectedSourceExists = + selectedSourceId != null && + sourceRows.some((source) => source.id === selectedSourceId); + const resolvedSourceId = selectedSourceExists + ? selectedSourceId + : (sourceRows[0]?.id ?? null); const documents = useQuery({ queryKey: ["source-documents", resolvedSourceId], @@ -24,102 +132,810 @@ export default function SourcesPage() { enabled: resolvedSourceId != null, }); - const selected = sources.data?.sources.find((source) => source.id === resolvedSourceId); + const documentRows = documents.data?.documents ?? []; + const resolvedDocumentId = + selectedDocumentId != null && + documentRows.some((doc) => doc.id === selectedDocumentId) + ? selectedDocumentId + : null; + + const fileContent = useQuery({ + queryKey: ["document-content", resolvedDocumentId], + queryFn: () => api.getDocumentContent(resolvedDocumentId!), + enabled: resolvedDocumentId != null, + }); + + const selected = sourceRows.find((source) => source.id === resolvedSourceId); + + const totals = useMemo(() => { + const rows = sources.data?.sources ?? []; + return rows.reduce( + (acc, source) => ({ + sources: acc.sources + 1, + documents: acc.documents + source.document_count, + chunks: acc.chunks + source.chunk_count, + }), + { sources: 0, documents: 0, chunks: 0 }, + ); + }, [sources.data?.sources]); + + const renameSource = useMutation({ + mutationFn: ({ sourceId, name }: { sourceId: number; name: string }) => + api.updateSource(sourceId, { name }, adminTokenValue), + onSuccess: async () => { + setEditingSourceId(null); + setSourceNameDraft(""); + await queryClient.invalidateQueries({ queryKey: ["sources"] }); + }, + }); + + const renameDocument = useMutation({ + mutationFn: ({ + documentId, + title, + }: { + documentId: number; + title: string; + }) => api.updateDocument(documentId, { title }, adminTokenValue), + onSuccess: async (doc) => { + setEditingDocumentId(null); + setDocumentTitleDraft(""); + await Promise.all([ + queryClient.invalidateQueries({ + queryKey: ["source-documents", doc.source_id], + }), + queryClient.invalidateQueries({ + queryKey: ["document-content", doc.id], + }), + ]); + }, + }); + + const updateDocumentContent = useMutation({ + mutationFn: ({ + documentId, + content, + }: { + documentId: number; + content: string; + }) => api.updateDocumentContent(documentId, { content }, adminTokenValue), + onSuccess: async (data) => { + setIsEditingContent(false); + setContentDraft(data.content ?? ""); + queryClient.setQueryData(["document-content", data.document.id], data); + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["sources"] }), + queryClient.invalidateQueries({ + queryKey: ["source-documents", data.document.source_id], + }), + queryClient.invalidateQueries({ + queryKey: ["document-content", data.document.id], + }), + ]); + }, + }); + + const deleteSource = useMutation({ + mutationFn: ({ sourceId }: { sourceId: number }) => + api.deleteSource(sourceId, adminTokenValue), + onSuccess: async (data) => { + setDeleteTarget(null); + setDeleteConfirm(""); + if (resolvedSourceId === data.source_id) { + setSelectedSourceId(null); + setSelectedDocumentId(null); + setIsEditingContent(false); + setContentDraft(""); + } + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["sources"] }), + queryClient.invalidateQueries({ queryKey: ["source-documents"] }), + ]); + }, + }); + + const deleteDocument = useMutation({ + mutationFn: ({ documentId }: { documentId: number }) => + api.deleteDocument(documentId, adminTokenValue), + onSuccess: async (data) => { + setDeleteTarget(null); + setDeleteConfirm(""); + if (selectedDocumentId === data.document_id) { + setSelectedDocumentId(null); + setIsEditingContent(false); + setContentDraft(""); + } + await Promise.all([ + queryClient.invalidateQueries({ queryKey: ["sources"] }), + queryClient.invalidateQueries({ + queryKey: ["source-documents", data.source_id], + }), + queryClient.invalidateQueries({ + queryKey: ["document-content", data.document_id], + }), + ]); + }, + }); + + const actionError = + renameSource.error ?? + renameDocument.error ?? + updateDocumentContent.error ?? + deleteSource.error ?? + deleteDocument.error; + + function startSourceRename(source: SourceSummary) { + renameSource.reset(); + setEditingSourceId(source.id); + setSourceNameDraft(source.name); + } + + function startDocumentRename(doc: DocumentSummary) { + renameDocument.reset(); + setEditingDocumentId(doc.id); + setDocumentTitleDraft(doc.title); + } + + function requestSourceDelete(source: SourceSummary) { + deleteSource.reset(); + setDeleteTarget({ + kind: "source", + id: source.id, + label: source.name, + documents: source.document_count, + }); + setDeleteConfirm(""); + } + + function requestDocumentDelete(doc: DocumentSummary) { + deleteDocument.reset(); + setDeleteTarget({ + kind: "document", + id: doc.id, + sourceId: doc.source_id, + label: doc.title, + }); + setDeleteConfirm(""); + } + + function saveSourceRename() { + const name = sourceNameDraft.trim(); + if (!editingSourceId || !name || !hasAdminToken) return; + renameSource.mutate({ sourceId: editingSourceId, name }); + } + + function saveDocumentRename() { + const title = documentTitleDraft.trim(); + if (!editingDocumentId || !title || !hasAdminToken) return; + renameDocument.mutate({ documentId: editingDocumentId, title }); + } + + function selectSource(sourceId: number) { + setSelectedSourceId(sourceId); + setSelectedDocumentId(null); + setEditingDocumentId(null); + setIsEditingContent(false); + setContentDraft(""); + updateDocumentContent.reset(); + } + + function selectDocument(documentId: number) { + setSelectedDocumentId(documentId); + setIsEditingContent(false); + setContentDraft(""); + updateDocumentContent.reset(); + } + + function confirmDelete() { + if ( + !deleteTarget || + deleteConfirm.trim() !== String(deleteTarget.id) || + !hasAdminToken + ) + return; + if (deleteTarget.kind === "source") { + deleteSource.mutate({ sourceId: deleteTarget.id }); + } else { + deleteDocument.mutate({ documentId: deleteTarget.id }); + } + } + + const deleting = deleteSource.isPending || deleteDocument.isPending; + + function startContentEdit() { + if (!fileContent.data?.content || fileContent.data.truncated) return; + updateDocumentContent.reset(); + setContentDraft(fileContent.data.content); + setIsEditingContent(true); + } + + function saveContentEdit() { + if ( + resolvedDocumentId == null || + !hasAdminToken || + !contentDraft.trim() || + contentDraft === (fileContent.data?.content ?? "") + ) { + return; + } + updateDocumentContent.mutate({ + documentId: resolvedDocumentId, + content: contentDraft, + }); + } return ( + router.push("/ingest")} + className="w-full sm:w-auto" + > + Add New Sources + + + setAdminToken(event.target.value)} + placeholder="Required for rename/delete/save" + /> + +
+ } > -
- - +
+ + + +
+ + {actionError && ( + + )} + + {deleteTarget && ( +
+
+
+ +
+

+ Delete{" "} + {deleteTarget.kind === "source" ? "source folder" : "file"} # + {deleteTarget.id} +

+

+ {deleteTarget.kind === "source" + ? `${deleteTarget.label} and ${deleteTarget.documents} file${deleteTarget.documents === 1 ? "" : "s"} will be removed with their chunks and embeddings.` + : `${deleteTarget.label} will be removed with its chunks and embeddings.`} +

+
+
+
+ setDeleteConfirm(event.target.value)} + className="font-mono focus-visible:border-destructive focus-visible:ring-destructive/15" + placeholder={`Type ${deleteTarget.id} to confirm`} + /> +
+ { + setDeleteTarget(null); + setDeleteConfirm(""); + }} + > + Cancel + + + {" "} + {deleting ? "Deleting" : "Delete"} + +
+ {!hasAdminToken && ( +

+ Admin token required. +

+ )} +
+
+
+ )} + +
+ + {sources.isLoading && } {sources.error && !sources.isLoading && (
- +
)} {sources.data?.sources.length === 0 && ( - } title="No sources yet" body="Ingest a document to populate this list." /> + } + title="No sources yet" + body="Use Add New Sources to populate this list." + /> )} {sources.data && sources.data.sources.length > 0 && ( -
+
{sources.data.sources.map((source) => { const active = source.id === resolvedSourceId; + const editing = source.id === editingSourceId; return ( - + ) : ( +
+ +
+ startSourceRename(source)} + > + + + requestSourceDelete(source)} + > + + +
+
+ )} +
); })}
)} - - - {!resolvedSourceId && !sources.isLoading && ( - } title="Select a source" /> - )} - {documents.isLoading && } - {documents.error && !documents.isLoading && ( -
- -
- )} - {documents.data?.documents.length === 0 && ( - } title="No documents for this source" /> - )} - {documents.data && documents.data.documents.length > 0 && ( -
- {documents.data.documents.map((doc) => ( -
-
- {doc.status} -

{doc.title}

- #{doc.id} +
+ + + {selected.document_count} files + {selected.chunk_count} chunks
-
- {doc.chunk_count} chunks - - raw text {doc.raw_text_available ? "kept" : "purged"} + ) : undefined + } + /> + {!resolvedSourceId && !sources.isLoading && ( + } + title="Select a source" + /> + )} + {documents.isLoading && } + {documents.error && !documents.isLoading && ( +
+ +
+ )} + {documents.data?.documents.length === 0 && ( + } + title="No files in this source" + /> + )} + {documents.data && documents.data.documents.length > 0 && ( +
+ {documentRows.map((doc) => { + const active = doc.id === resolvedDocumentId; + const editing = doc.id === editingDocumentId; + return ( +
+ {editing ? ( +
+ + + setDocumentTitleDraft(event.target.value) + } + autoFocus + /> + +
+ setEditingDocumentId(null)} + > + Cancel + + + {" "} + {renameDocument.isPending ? "Saving" : "Save"} + +
+
+ ) : ( +
+ +
+ selectDocument(doc.id)} + > + + + startDocumentRename(doc)} + > + + + requestDocumentDelete(doc)} + > + + +
+
+ )} +
+ ); + })} +
+ )} + + + + + + {contentSourceLabel(fileContent.data.content_source)} - {doc.tags.map((tag) => ( - - {tag} - - ))} + {fileContent.data.content && !isEditingContent && ( + + Edit + + )}
-

- Ingested {formatDateTime(doc.ingested_at)} / {doc.content_type ?? "unknown type"} + ) : undefined + } + /> + {resolvedDocumentId == null && ( + } + title="No file selected" + body="Choose a file from the list to view stored text or indexed chunks." + /> + )} + {fileContent.isLoading && } + {fileContent.error && !fileContent.isLoading && ( +

+ +
+ )} + {fileContent.data && ( +
+
+
+ + {fileContent.data.document.status} + + #{fileContent.data.document.id} + + {fileContent.data.document.chunk_count} chunks + +
+

+ Source: {fileContent.data.source.name} / Updated{" "} + {formatDateTime(fileContent.data.document.updated_at)}

- ))} -
- )} - + {fileContent.data.content ? ( + isEditingContent ? ( +
+ +