diff --git a/.env.example b/.env.example index 4255850..b3c4124 100644 --- a/.env.example +++ b/.env.example @@ -51,13 +51,61 @@ # --- Tools ---------------------------------------------------------------- # Only tools listed here are dispatchable, regardless of what a # module implements. chat,code is the conservative default; add -# files/shell/git once you trust the sandboxing for your setup, and -# memory once you have an embedding server running (see below). -# ENABLED_TOOLS=chat,code,files,shell,git,memory +# files/shell/git once you trust the sandboxing for your setup, +# memory once you have an embedding server running (see below), and +# review/test/web_fetch/web_search/research as each of their own +# sections below is configured. +# ENABLED_TOOLS=chat,code,files,shell,git,memory,review,test,web_fetch,web_search,research # WORKSPACE_DIR=data/workspace # SHELL_TIMEOUT=30 # SHELL_ALLOWED_COMMANDS=ls,cat,head,tail,wc,grep,find,python3,pip,pytest +# --- Test/lint tool --------------------------------------------------------- +# Separate allowlist from the general shell tool above, on purpose -- +# see tools/test.py. +# TEST_TIMEOUT=60 +# TEST_ALLOWED_COMMANDS=pytest,ruff + +# --- Review graph (search a file, optionally run its tests, then review) -- +# No config of its own beyond ENABLED_TOOLS -- reuses WORKSPACE_DIR +# confinement (tools/review.py) and the test tool above when a +# test_path is given. Dispatchable from chat: "relis X et donne ton +# avis", or "relis X et lance ses tests dans Y". + +# --- Web fetch -------------------------------------------------------------- +# Fetches a URL you already know -- see tools/web_fetch.py for the +# SSRF guard (not configurable) and its known limitation on heavy, +# non-semantic sites. +# WEB_FETCH_TIMEOUT=15 +# WEB_FETCH_MAX_BYTES=2097152 +# Empty = any public domain is fetchable, subject to the SSRF guard. +# WEB_FETCH_ALLOWED_DOMAINS= + +# --- Web search (SearXNG) ---------------------------------------------------- +# Requires a self-hosted SearXNG instance -- not a cloud search API. +# SearXNG's own settings.yml needs "json" added to search.formats +# (disabled by default upstream to discourage scraping public +# instances; fine to enable on a private, self-hosted one). +# Running under podman/Docker? Same host.containers.internal / +# host.docker.internal note as LLAMA_CPP_URL above applies here. +# Only returns a ranked links/snippets list, no synthesis -- for an +# actual answer, use "research" below instead. +# SEARXNG_URL=http://127.0.0.1:8888 +# SEARXNG_TIMEOUT=10 +# SEARXNG_MAX_RESULTS=5 + +# --- Research graph (search -> fetch top N -> synthesize) ------------------ +# The default choice for an actual answer/summary about something +# current ("actualités", "quoi de neuf sur X") -- a single +# dispatchable call that runs search, fetches the top results, and +# synthesizes one answer internally (see graphs/research.py). Exists +# specifically because chaining "web_search" into a router-decided +# second step proved unreliable with small local models. Requires the +# same SearXNG instance as web_search above (doesn't need +# "web_search" itself in ENABLED_TOOLS, calls it directly). +# RESEARCH_FETCH_TOP_N=3 +# RESEARCH_FETCH_CHARS_PER_RESULT=1500 + # --- Memory --------------------------------------------------------------- # MEMORY_ENABLED=true # MEMORY_FILE=data/memory.json diff --git a/README.md b/README.md index 7f0b5e9..e072d6e 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,17 @@ User Input LLM Router (structured JSON decision) ↓ Tool Dispatcher - ├── chat (conversational response) - ├── code (code generation) - ├── files (sandboxed read/write/list) - ├── shell (sandboxed subprocess) - └── git (read-only git operations) + ├── chat (conversational response) + ├── code (code generation) + ├── files (sandboxed read/write/list) + ├── shell (sandboxed subprocess) + ├── git (read-only git operations) + ├── memory (remember/recall, vector search) + ├── test (sandboxed pytest/ruff runner) + ├── review (read a file, optionally test it, analyze) + ├── web_fetch (fetch a known URL) + ├── web_search (SearXNG links/snippets, no synthesis) + └── research (search → fetch → synthesize, one call) ``` The model must output a strict JSON instruction (`{"tool": "...", "content": "..."}`) @@ -108,7 +114,9 @@ src/forge/ ├── graph.py # Node / Edge / Graph execution engine ├── graphs/ │ ├── default.py # router → dispatch → fallback (drop-in for Orchestrator) -│ └── review.py # read_file → llm_review (chains filesystem + LLM) +│ ├── review.py # read_file → [run_tests] → llm_review (optional test_path adds the middle step) +│ └── research.py # search → fetch top N → synthesize, one deterministic call (v3.10) +├── text_cleaning.py # shared plain-text response cleaning (review.py + research.py) │ ├── router/ │ ├── prompt.py # router prompt template — isolated; nothing else builds prompts @@ -120,13 +128,18 @@ src/forge/ │ ├── code.py │ ├── files.py # sandboxed read/write/list within WORKSPACE_DIR │ ├── shell.py # sandboxed subprocess within WORKSPACE_DIR + allowlist -│ ├── git.py # read-only git operations (status/diff/log/show/branch) -│ └── memory.py # router-dispatchable remember/recall (v3.7) — same rag.py backend +│ ├── git.py # read-only git operations (status/diff/log/show/branch) — no write counterpart, by design +│ ├── memory.py # router-dispatchable remember/recall (v3.7) — same rag.py backend +│ ├── test.py # sandboxed pytest/ruff runner, own allowlist (v3.10) +│ ├── review.py # dispatchable wrapper around graphs/review.py (v3.10) +│ ├── web_fetch.py # fetch a known URL, SSRF-guarded (v3.10) +│ ├── web_search.py # SearXNG-backed search, links/snippets only (v3.10) +│ └── research.py # dispatchable wrapper around graphs/research.py (v3.10) │ ├── memory.py # JSON-backed rolling conversation history + key/value facts ├── rag.py # SQLite-vec vector memory for decisions/todos (v3.7) — separate concern from memory.py ├── api.py # FastAPI HTTP server (chat, review, run, traces, tools, remember, search) -├── cli.py # forge review / forge replay +├── cli.py # forge review [--tests ] / forge replay ├── main.py # REPL — !clear, !trace, !remember, !recall, !help │ └── providers/ @@ -228,6 +241,10 @@ podman run --rm --env-file .env.local \ -v $(pwd):/workspace forge-core \ python -m forge.cli review src/forge/main.py "Que peut-on améliorer ?" +# Review a file and run its tests first (v3.10) -- test output becomes +# primary evidence for the review, not just the code itself +python -m forge.cli review src/forge/graph.py --tests tests/test_graph.py + # Replay a past execution trace python -m forge.cli replay ``` @@ -241,7 +258,7 @@ python -m forge.cli replay | `GET` | `/` | open | Web UI | | `GET` | `/health` | open | Provider + model info (for `llama_cpp`, the actually-loaded model, queried live from llama-server — see below) | | `POST` | `/chat` | optional | Single conversation turn | -| `POST` | `/review` | optional | File content analysis | +| `POST` | `/review` | optional | File content analysis, optionally running its tests first (`test_path` field, v3.10) | | `POST` | `/run` | optional | Run any graph by name | | `GET` | `/tools` | optional | Active tools + available graphs | | `GET` | `/traces?n=10` | optional | Recent execution traces | @@ -314,6 +331,16 @@ e.g. behind a proxy that already rate-limits. | `EMBEDDING_DIM` | Embedding vector dimension, must match the served model | `1024` | | `EMBEDDING_TIMEOUT` | HTTP timeout for embedding requests (seconds) | `30` | | `RAG_DB_FILE` | Path to the SQLite-vec vector memory file | `data/forge_rag.db` | +| `TEST_TIMEOUT` | Max seconds for a test/lint tool command | `60` | +| `TEST_ALLOWED_COMMANDS` | Comma-separated command allowlist for the test tool — separate from `SHELL_ALLOWED_COMMANDS` on purpose | `pytest,ruff` | +| `WEB_FETCH_TIMEOUT` | HTTP timeout for `web_fetch` requests (seconds) | `15` | +| `WEB_FETCH_MAX_BYTES` | Raw response byte cap before truncation | `2097152` (2 MiB) | +| `WEB_FETCH_ALLOWED_DOMAINS` | Optional domain allowlist — empty means any public domain, subject to the (non-configurable) SSRF guard | *(empty)* | +| `SEARXNG_URL` | Self-hosted SearXNG instance for `web_search`/`research` — not a cloud API | `http://127.0.0.1:8888` | +| `SEARXNG_TIMEOUT` | HTTP timeout for SearXNG requests (seconds) | `10` | +| `SEARXNG_MAX_RESULTS` | Max results returned per search | `5` | +| `RESEARCH_FETCH_TOP_N` | How many top search results `research` fetches in full before synthesizing | `3` | +| `RESEARCH_FETCH_CHARS_PER_RESULT` | Per-result fetched-content cap fed into the synthesis prompt | `1500` | --- @@ -325,7 +352,13 @@ e.g. behind a proxy that already rate-limits. | `code` | default | Code generation | | `files` | `ENABLED_TOOLS=chat,code,files` | Sandboxed read/write/list within `WORKSPACE_DIR` | | `shell` | `ENABLED_TOOLS=chat,code,shell` | Subprocess execution within `WORKSPACE_DIR` + `SHELL_ALLOWED_COMMANDS` | -| `git` | `ENABLED_TOOLS=chat,code,git` | Read-only git operations (status, diff, log, show, branch) | +| `git` | `ENABLED_TOOLS=chat,code,git` | Read-only git operations (status, diff, log, show, branch) — deliberately never gains a write counterpart reachable by the router: a commit/push has a real cost if the router hallucinates, so any git write stays a separate, human-confirmed flow outside tool dispatch, not a router decision | +| `memory` | `ENABLED_TOOLS=chat,code,memory` | Router-dispatchable RAG remember/recall (v3.7) | +| `test` | `ENABLED_TOOLS=chat,code,test` | Sandboxed pytest/ruff runner, own allowlist (`TEST_ALLOWED_COMMANDS`) separate from the shell tool's | +| `review` | `ENABLED_TOOLS=chat,code,review` | Reads a file (optionally runs its tests first) and returns an LLM analysis — "relis X et donne ton avis", not just "lis X" (see [Router reachability](#tools) note below on that exact ambiguity) | +| `web_fetch` | `ENABLED_TOOLS=chat,code,web_fetch` | Fetches a URL you already know — no search capability, SSRF-guarded, best-effort HTML→text extraction | +| `web_search` | `ENABLED_TOOLS=chat,code,web_search` | Ranked links/snippets from a self-hosted SearXNG instance — no synthesis, just the list | +| `research` | `ENABLED_TOOLS=chat,code,research` | Search → fetch top results → synthesize one answer, as a single deterministic call (see below) | A tool is only dispatchable if it has a `run()` function **and** appears in `ENABLED_TOOLS`. Implementing `run()` in a module is not enough — the opt-in is intentional for tools with side effects. @@ -355,6 +388,19 @@ completion field, or to rule it out while debugging — the prompt-engineering + chain underneath it all is unchanged and still does the same job on its own, just with a higher failure rate on a stressed prompt. +**Why `research` exists alongside `web_search` (v3.10):** a plain search only returns links and +snippets — turning that into an actual synthesized answer needs a second step (fetch a promising +result, then have the model write a real answer from it). Asking the router to decide that second +step itself proved unreliable in practice with a small local model: even with an explicit worked +JSON example showing exactly what to do next, it would sometimes just repeat the identical search +call instead, tripping the loop guard. Disabling `LLAMA_CPP_CACHE_PROMPT` and reproducing the same +failure ruled out a KV-cache bug — this is a genuine limit at multi-step self-correction for this +model class, not a fixable prompt or infra issue. `research` (`graphs/research.py`) removes the +decision from the router's hands entirely: search → fetch the top `RESEARCH_FETCH_TOP_N` results → +one synthesis call, run as a fixed sequence inside a single dispatchable call, the same pattern +already used by the `review` graph. `web_search` stays for when the user genuinely wants a list of +links/sources rather than an answer. + **Why `/chat` isn't streamed (yet):** for `tool="chat"`, the router's single LLM call already *is* the answer — `content` in `{"tool":"chat","content":"..."}` is generated in the same call as the routing decision, and `tools/chat.py` just returns it unchanged. Streaming that content would @@ -535,7 +581,10 @@ Same commands locally, after `pip install -r requirements-dev.txt`. | **v3.4** | done | Portfolio: architecture diagram, `.env.example`, LinkedIn writeup | | **v3.5** | done | Test coverage (llm/cli/trace: 26-39% → 98-100%), router reachable to files/shell/git, API rate limiting | | **v3.6** | done | Response quality: GBNF grammar-constrained decoding for llama.cpp | -| **v3.7** | current | Vector memory / RAG: SQLite-vec, `/remember` + `/search`, `!remember`/`!recall` REPL commands, a router-dispatchable `memory` tool, Qwen3-Embedding-0.6B | +| **v3.7** | done | Vector memory / RAG: SQLite-vec, `/remember` + `/search`, `!remember`/`!recall` REPL commands, a router-dispatchable `memory` tool, Qwen3-Embedding-0.6B | +| **v3.8** | done | Prompt-cache reliability: pinned llama-server slot, `MEMORY_MAX_HISTORY` raised to stop a sliding window from fighting KV-cache reuse — root-caused a remaining cache-reuse gap to the served model's own hybrid architecture, not Forge | +| **v3.9** | done | Context compaction + drawer: `rag_pointer`/`llm_summary` strategies, pin/unpin, `/history` `/drawer` `/compact` endpoints, `!compact` REPL command, files write-diff | +| **v3.10** | current | Hardening + new tools: dedicated `test` tool, `web_fetch` (SSRF-guarded), `web_search` + `research` (self-hosted SearXNG), review graph gains an optional test-run step and chat-dispatch; router disambiguation fixes (files vs review, tool descriptions/examples for every new tool) found through real usage | --- diff --git a/requirements-dev.txt b/requirements-dev.txt index aa18872..e906922 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,5 @@ pytest ruff httpx +requests-mock +httpx2 diff --git a/src/forge/api.py b/src/forge/api.py index 1c8f001..fe0179e 100644 --- a/src/forge/api.py +++ b/src/forge/api.py @@ -95,6 +95,7 @@ class ReviewRequest(BaseModel): content: str # file content (not a path) filename: str = "untitled" question: str = "Que peut-on améliorer ?" + test_path: str | None = None # optional, run these tests before reviewing class ChatResponse(BaseModel): @@ -235,7 +236,13 @@ async def review(req: ReviewRequest): from forge.graphs.review import run as review_run - # Write the content to a temp file so the review graph can read it + # Write the content to a temp file so the review graph can read it. + # Note: test_path (if given) is resolved relative to WORKSPACE_DIR + # by the test tool, NOT relative to this temp file -- running + # tests against submitted content only makes sense when that + # content already corresponds to a file inside the workspace + # (e.g. reviewing a workspace file's current content with its + # existing test suite), not for arbitrary pasted snippets. suffix = Path(req.filename).suffix or ".txt" with tempfile.NamedTemporaryFile( mode="w", suffix=suffix, delete=False, encoding="utf-8" @@ -244,7 +251,7 @@ async def review(req: ReviewRequest): tmp_path = f.name try: - output = await _run_in_thread(review_run, tmp_path, req.question) + output = await _run_in_thread(review_run, tmp_path, req.question, req.test_path) finally: os.unlink(tmp_path) diff --git a/src/forge/cli.py b/src/forge/cli.py index 9ac63d8..7b83906 100644 --- a/src/forge/cli.py +++ b/src/forge/cli.py @@ -2,8 +2,13 @@ Forge CLI — command-line entry points beyond the REPL. Available commands: - forge review [question] — review a file with the LLM - forge replay — replay a past execution trace + forge review [question] [--tests ] — review a file + with the LLM, + optionally + running its + tests first + forge replay — replay a past + execution trace Run from container: podman run --rm --env-file .env.local \\ @@ -12,6 +17,7 @@ Or directly: PYTHONPATH=src python -m forge.cli review src/forge/main.py + PYTHONPATH=src python -m forge.cli review src/forge/graph.py --tests tests/test_graph.py """ import sys @@ -19,16 +25,25 @@ def _cmd_review(args: list[str]) -> int: if not args: - print("Usage: forge review [question]", file=sys.stderr) + print("Usage: forge review [question] [--tests ]", file=sys.stderr) return 1 + test_path = None + if "--tests" in args: + idx = args.index("--tests") + if idx + 1 >= len(args): + print("Usage: --tests requires a path argument", file=sys.stderr) + return 1 + test_path = args[idx + 1] + args = args[:idx] + args[idx + 2 :] + file_path = args[0] question = " ".join(args[1:]) if len(args) > 1 else "Que peut-on améliorer ?" from forge.graphs.review import run print(f"Reviewing {file_path!r}…\n") - result = run(file_path, question=question) + result = run(file_path, question=question, test_path=test_path) print(result) return 0 diff --git a/src/forge/config.py b/src/forge/config.py index e1d93f9..b1fd1a7 100644 --- a/src/forge/config.py +++ b/src/forge/config.py @@ -117,6 +117,19 @@ def _bool(name: str, default: str = "false") -> bool: if c.strip() } +# --- Test/lint tool ---------------------------------------------------------- +# Separate from SHELL_ALLOWED_COMMANDS on purpose: the test tool has its own +# narrower allowlist so "run the tests" / "lint this" stay first-class router +# intents with a purpose-built safety boundary, independent of whatever the +# general shell tool happens to allow. +TEST_TIMEOUT = int(os.getenv("TEST_TIMEOUT", "60")) +_default_test_cmds = "pytest,ruff" +TEST_ALLOWED_COMMANDS: set[str] = { + c.strip() + for c in os.getenv("TEST_ALLOWED_COMMANDS", _default_test_cmds).split(",") + if c.strip() +} + # --- Tool allowlist ---------------------------------------------------------# A module exposing run() in src/forge/tools/ is NOT dispatchable just # because it exists. It must also be explicitly listed here. This is # the guard that matters once files.py / git.py / shell.py stop being @@ -164,3 +177,53 @@ def _bool(name: str, default: str = "false") -> bool: EMBEDDING_DIM = int(os.getenv("EMBEDDING_DIM", "1024")) EMBEDDING_TIMEOUT = int(os.getenv("EMBEDDING_TIMEOUT", "30")) RAG_DB_FILE = os.getenv("RAG_DB_FILE", "data/forge_rag.db") + +# --- Web fetch tool ---------------------------------------------------------- +# WEB_FETCH_ALLOWED_DOMAINS is empty by default (any public domain is +# fetchable) -- the SSRF guard in tools/web_fetch.py (blocking private/ +# loopback/link-local resolved IPs) is NOT configurable and always applies, +# regardless of this allowlist. This matters specifically because Forge +# itself sits on a home network (NiPoGi behind WireGuard) that a +# router-hallucinated URL must never be able to reach. +# +# Known limitation, not fixed by raising this value alone: heavy +# corporate portals (observed live: boursorama.com) often wrap large +# navigation megamenus in generic
s instead of semantic +#