v3.10 — Hardening + new tools (test, web_fetch, web_search, research)

.env.example

@@ -51,13 +51,61 @@
 # --- Tools ----------------------------------------------------------------
 # Only tools listed here are dispatchable, regardless of what a
 # module implements. chat,code is the conservative default; add
-# files/shell/git once you trust the sandboxing for your setup, and
-# memory once you have an embedding server running (see below).
-# ENABLED_TOOLS=chat,code,files,shell,git,memory
+# files/shell/git once you trust the sandboxing for your setup,
+# memory once you have an embedding server running (see below), and
+# review/test/web_fetch/web_search/research as each of their own
+# sections below is configured.
+# ENABLED_TOOLS=chat,code,files,shell,git,memory,review,test,web_fetch,web_search,research
 # WORKSPACE_DIR=data/workspace
 # SHELL_TIMEOUT=30
 # SHELL_ALLOWED_COMMANDS=ls,cat,head,tail,wc,grep,find,python3,pip,pytest
 
+# --- Test/lint tool ---------------------------------------------------------
+# Separate allowlist from the general shell tool above, on purpose --
+# see tools/test.py.
+# TEST_TIMEOUT=60
+# TEST_ALLOWED_COMMANDS=pytest,ruff
+
+# --- Review graph (search a file, optionally run its tests, then review) --
+# No config of its own beyond ENABLED_TOOLS -- reuses WORKSPACE_DIR
+# confinement (tools/review.py) and the test tool above when a
+# test_path is given. Dispatchable from chat: "relis X et donne ton
+# avis", or "relis X et lance ses tests dans Y".
+
+# --- Web fetch --------------------------------------------------------------
+# Fetches a URL you already know -- see tools/web_fetch.py for the
+# SSRF guard (not configurable) and its known limitation on heavy,
+# non-semantic sites.
+# WEB_FETCH_TIMEOUT=15
+# WEB_FETCH_MAX_BYTES=2097152
+# Empty = any public domain is fetchable, subject to the SSRF guard.
+# WEB_FETCH_ALLOWED_DOMAINS=
+
+# --- Web search (SearXNG) ----------------------------------------------------
+# Requires a self-hosted SearXNG instance -- not a cloud search API.
+# SearXNG's own settings.yml needs "json" added to search.formats
+# (disabled by default upstream to discourage scraping public
+# instances; fine to enable on a private, self-hosted one).
+# Running under podman/Docker? Same host.containers.internal /
+# host.docker.internal note as LLAMA_CPP_URL above applies here.
+# Only returns a ranked links/snippets list, no synthesis -- for an
+# actual answer, use "research" below instead.
+# SEARXNG_URL=http://127.0.0.1:8888
+# SEARXNG_TIMEOUT=10
+# SEARXNG_MAX_RESULTS=5
+
+# --- Research graph (search -> fetch top N -> synthesize) ------------------
+# The default choice for an actual answer/summary about something
+# current ("actualités", "quoi de neuf sur X") -- a single
+# dispatchable call that runs search, fetches the top results, and
+# synthesizes one answer internally (see graphs/research.py). Exists
+# specifically because chaining "web_search" into a router-decided
+# second step proved unreliable with small local models. Requires the
+# same SearXNG instance as web_search above (doesn't need
+# "web_search" itself in ENABLED_TOOLS, calls it directly).
+# RESEARCH_FETCH_TOP_N=3
+# RESEARCH_FETCH_CHARS_PER_RESULT=1500
+
 # --- Memory ---------------------------------------------------------------
 # MEMORY_ENABLED=true
 # MEMORY_FILE=data/memory.json

README.md

@@ -16,11 +16,17 @@ User Input
 LLM Router  (structured JSON decision)
    ↓
 Tool Dispatcher
-   ├── chat     (conversational response)
-   ├── code     (code generation)
-   ├── files    (sandboxed read/write/list)
-   ├── shell    (sandboxed subprocess)
-   └── git      (read-only git operations)
+   ├── chat        (conversational response)
+   ├── code        (code generation)
+   ├── files       (sandboxed read/write/list)
+   ├── shell       (sandboxed subprocess)
+   ├── git         (read-only git operations)
+   ├── memory      (remember/recall, vector search)
+   ├── test        (sandboxed pytest/ruff runner)
+   ├── review      (read a file, optionally test it, analyze)
+   ├── web_fetch   (fetch a known URL)
+   ├── web_search  (SearXNG links/snippets, no synthesis)
+   └── research    (search → fetch → synthesize, one call)
 ```
 
 The model must output a strict JSON instruction (`{"tool": "...", "content": "..."}`)
@@ -108,7 +114,9 @@ src/forge/
 ├── graph.py             # Node / Edge / Graph execution engine
 ├── graphs/
 │   ├── default.py       # router → dispatch → fallback (drop-in for Orchestrator)
-│   └── review.py        # read_file → llm_review (chains filesystem + LLM)
+│   ├── review.py        # read_file → [run_tests] → llm_review (optional test_path adds the middle step)
+│   └── research.py      # search → fetch top N → synthesize, one deterministic call (v3.10)
+├── text_cleaning.py     # shared plain-text response cleaning (review.py + research.py)
 │
 ├── router/
 │   ├── prompt.py        # router prompt template — isolated; nothing else builds prompts
@@ -120,13 +128,18 @@ src/forge/
 │   ├── code.py
 │   ├── files.py         # sandboxed read/write/list within WORKSPACE_DIR
 │   ├── shell.py         # sandboxed subprocess within WORKSPACE_DIR + allowlist
-│   ├── git.py           # read-only git operations (status/diff/log/show/branch)
-│   └── memory.py        # router-dispatchable remember/recall (v3.7) — same rag.py backend
+│   ├── git.py           # read-only git operations (status/diff/log/show/branch) — no write counterpart, by design
+│   ├── memory.py        # router-dispatchable remember/recall (v3.7) — same rag.py backend
+│   ├── test.py          # sandboxed pytest/ruff runner, own allowlist (v3.10)
+│   ├── review.py        # dispatchable wrapper around graphs/review.py (v3.10)
+│   ├── web_fetch.py      # fetch a known URL, SSRF-guarded (v3.10)
+│   ├── web_search.py    # SearXNG-backed search, links/snippets only (v3.10)
+│   └── research.py      # dispatchable wrapper around graphs/research.py (v3.10)
 │
 ├── memory.py            # JSON-backed rolling conversation history + key/value facts
 ├── rag.py               # SQLite-vec vector memory for decisions/todos (v3.7) — separate concern from memory.py
 ├── api.py               # FastAPI HTTP server (chat, review, run, traces, tools, remember, search)
-├── cli.py               # forge review <file> / forge replay <run_id>
+├── cli.py               # forge review <file> [--tests <path>] / forge replay <run_id>
 ├── main.py              # REPL — !clear, !trace, !remember, !recall, !help
 │
 └── providers/
@@ -228,6 +241,10 @@ podman run --rm --env-file .env.local \
   -v $(pwd):/workspace forge-core \
   python -m forge.cli review src/forge/main.py "Que peut-on améliorer ?"
 
+# Review a file and run its tests first (v3.10) -- test output becomes
+# primary evidence for the review, not just the code itself
+python -m forge.cli review src/forge/graph.py --tests tests/test_graph.py
+
 # Replay a past execution trace
 python -m forge.cli replay <run_id>
 ```
@@ -241,7 +258,7 @@ python -m forge.cli replay <run_id>
 | `GET` | `/` | open | Web UI |
 | `GET` | `/health` | open | Provider + model info (for `llama_cpp`, the actually-loaded model, queried live from llama-server — see below) |
 | `POST` | `/chat` | optional | Single conversation turn |
-| `POST` | `/review` | optional | File content analysis |
+| `POST` | `/review` | optional | File content analysis, optionally running its tests first (`test_path` field, v3.10) |
 | `POST` | `/run` | optional | Run any graph by name |
 | `GET` | `/tools` | optional | Active tools + available graphs |
 | `GET` | `/traces?n=10` | optional | Recent execution traces |
@@ -314,6 +331,16 @@ e.g. behind a proxy that already rate-limits.
 | `EMBEDDING_DIM` | Embedding vector dimension, must match the served model | `1024` |
 | `EMBEDDING_TIMEOUT` | HTTP timeout for embedding requests (seconds) | `30` |
 | `RAG_DB_FILE` | Path to the SQLite-vec vector memory file | `data/forge_rag.db` |
+| `TEST_TIMEOUT` | Max seconds for a test/lint tool command | `60` |
+| `TEST_ALLOWED_COMMANDS` | Comma-separated command allowlist for the test tool — separate from `SHELL_ALLOWED_COMMANDS` on purpose | `pytest,ruff` |
+| `WEB_FETCH_TIMEOUT` | HTTP timeout for `web_fetch` requests (seconds) | `15` |
+| `WEB_FETCH_MAX_BYTES` | Raw response byte cap before truncation | `2097152` (2 MiB) |
+| `WEB_FETCH_ALLOWED_DOMAINS` | Optional domain allowlist — empty means any public domain, subject to the (non-configurable) SSRF guard | *(empty)* |
+| `SEARXNG_URL` | Self-hosted SearXNG instance for `web_search`/`research` — not a cloud API | `http://127.0.0.1:8888` |
+| `SEARXNG_TIMEOUT` | HTTP timeout for SearXNG requests (seconds) | `10` |
+| `SEARXNG_MAX_RESULTS` | Max results returned per search | `5` |
+| `RESEARCH_FETCH_TOP_N` | How many top search results `research` fetches in full before synthesizing | `3` |
+| `RESEARCH_FETCH_CHARS_PER_RESULT` | Per-result fetched-content cap fed into the synthesis prompt | `1500` |
 
 ---
 
@@ -325,7 +352,13 @@ e.g. behind a proxy that already rate-limits.
 | `code` | default | Code generation |
 | `files` | `ENABLED_TOOLS=chat,code,files` | Sandboxed read/write/list within `WORKSPACE_DIR` |
 | `shell` | `ENABLED_TOOLS=chat,code,shell` | Subprocess execution within `WORKSPACE_DIR` + `SHELL_ALLOWED_COMMANDS` |
-| `git` | `ENABLED_TOOLS=chat,code,git` | Read-only git operations (status, diff, log, show, branch) |
+| `git` | `ENABLED_TOOLS=chat,code,git` | Read-only git operations (status, diff, log, show, branch) — deliberately never gains a write counterpart reachable by the router: a commit/push has a real cost if the router hallucinates, so any git write stays a separate, human-confirmed flow outside tool dispatch, not a router decision |
+| `memory` | `ENABLED_TOOLS=chat,code,memory` | Router-dispatchable RAG remember/recall (v3.7) |
+| `test` | `ENABLED_TOOLS=chat,code,test` | Sandboxed pytest/ruff runner, own allowlist (`TEST_ALLOWED_COMMANDS`) separate from the shell tool's |
+| `review` | `ENABLED_TOOLS=chat,code,review` | Reads a file (optionally runs its tests first) and returns an LLM analysis — "relis X et donne ton avis", not just "lis X" (see [Router reachability](#tools) note below on that exact ambiguity) |
+| `web_fetch` | `ENABLED_TOOLS=chat,code,web_fetch` | Fetches a URL you already know — no search capability, SSRF-guarded, best-effort HTML→text extraction |
+| `web_search` | `ENABLED_TOOLS=chat,code,web_search` | Ranked links/snippets from a self-hosted SearXNG instance — no synthesis, just the list |
+| `research` | `ENABLED_TOOLS=chat,code,research` | Search → fetch top results → synthesize one answer, as a single deterministic call (see below) |
 
 A tool is only dispatchable if it has a `run()` function **and** appears in `ENABLED_TOOLS`.
 Implementing `run()` in a module is not enough — the opt-in is intentional for tools with side effects.
@@ -355,6 +388,19 @@ completion field, or to rule it out while debugging — the prompt-engineering +
 chain underneath it all is unchanged and still does the same job on its own, just with a higher
 failure rate on a stressed prompt.
 
+**Why `research` exists alongside `web_search` (v3.10):** a plain search only returns links and
+snippets — turning that into an actual synthesized answer needs a second step (fetch a promising
+result, then have the model write a real answer from it). Asking the router to decide that second
+step itself proved unreliable in practice with a small local model: even with an explicit worked
+JSON example showing exactly what to do next, it would sometimes just repeat the identical search
+call instead, tripping the loop guard. Disabling `LLAMA_CPP_CACHE_PROMPT` and reproducing the same
+failure ruled out a KV-cache bug — this is a genuine limit at multi-step self-correction for this
+model class, not a fixable prompt or infra issue. `research` (`graphs/research.py`) removes the
+decision from the router's hands entirely: search → fetch the top `RESEARCH_FETCH_TOP_N` results →
+one synthesis call, run as a fixed sequence inside a single dispatchable call, the same pattern
+already used by the `review` graph. `web_search` stays for when the user genuinely wants a list of
+links/sources rather than an answer.
+
 **Why `/chat` isn't streamed (yet):** for `tool="chat"`, the router's single LLM call already
 *is* the answer — `content` in `{"tool":"chat","content":"..."}` is generated in the same call as
 the routing decision, and `tools/chat.py` just returns it unchanged. Streaming that content would
@@ -535,7 +581,10 @@ Same commands locally, after `pip install -r requirements-dev.txt`.
 | **v3.4** | done | Portfolio: architecture diagram, `.env.example`, LinkedIn writeup |
 | **v3.5** | done | Test coverage (llm/cli/trace: 26-39% → 98-100%), router reachable to files/shell/git, API rate limiting |
 | **v3.6** | done | Response quality: GBNF grammar-constrained decoding for llama.cpp |
-| **v3.7** | current | Vector memory / RAG: SQLite-vec, `/remember` + `/search`, `!remember`/`!recall` REPL commands, a router-dispatchable `memory` tool, Qwen3-Embedding-0.6B |
+| **v3.7** | done | Vector memory / RAG: SQLite-vec, `/remember` + `/search`, `!remember`/`!recall` REPL commands, a router-dispatchable `memory` tool, Qwen3-Embedding-0.6B |
+| **v3.8** | done | Prompt-cache reliability: pinned llama-server slot, `MEMORY_MAX_HISTORY` raised to stop a sliding window from fighting KV-cache reuse — root-caused a remaining cache-reuse gap to the served model's own hybrid architecture, not Forge |
+| **v3.9** | done | Context compaction + drawer: `rag_pointer`/`llm_summary` strategies, pin/unpin, `/history` `/drawer` `/compact` endpoints, `!compact` REPL command, files write-diff |
+| **v3.10** | current | Hardening + new tools: dedicated `test` tool, `web_fetch` (SSRF-guarded), `web_search` + `research` (self-hosted SearXNG), review graph gains an optional test-run step and chat-dispatch; router disambiguation fixes (files vs review, tool descriptions/examples for every new tool) found through real usage |
 
 ---
 

requirements-dev.txt

@@ -1,3 +1,5 @@
 pytest
 ruff
 httpx
+requests-mock
+httpx2

src/forge/api.py

@@ -95,6 +95,7 @@ class ReviewRequest(BaseModel):
     content: str  # file content (not a path)
     filename: str = "untitled"
     question: str = "Que peut-on améliorer ?"
+    test_path: str | None = None  # optional, run these tests before reviewing
 
 
 class ChatResponse(BaseModel):
@@ -235,7 +236,13 @@ async def review(req: ReviewRequest):
 
     from forge.graphs.review import run as review_run
 
-    # Write the content to a temp file so the review graph can read it
+    # Write the content to a temp file so the review graph can read it.
+    # Note: test_path (if given) is resolved relative to WORKSPACE_DIR
+    # by the test tool, NOT relative to this temp file -- running
+    # tests against submitted content only makes sense when that
+    # content already corresponds to a file inside the workspace
+    # (e.g. reviewing a workspace file's current content with its
+    # existing test suite), not for arbitrary pasted snippets.
     suffix = Path(req.filename).suffix or ".txt"
     with tempfile.NamedTemporaryFile(
         mode="w", suffix=suffix, delete=False, encoding="utf-8"
@@ -244,7 +251,7 @@ async def review(req: ReviewRequest):
         tmp_path = f.name
 
     try:
-        output = await _run_in_thread(review_run, tmp_path, req.question)
+        output = await _run_in_thread(review_run, tmp_path, req.question, req.test_path)
     finally:
         os.unlink(tmp_path)
 

src/forge/cli.py

@@ -2,8 +2,13 @@
 Forge CLI — command-line entry points beyond the REPL.
 
 Available commands:
-  forge review <file> [question]   — review a file with the LLM
-  forge replay <run_id>            — replay a past execution trace
+  forge review <file> [question] [--tests <path>]   — review a file
+                                                        with the LLM,
+                                                        optionally
+                                                        running its
+                                                        tests first
+  forge replay <run_id>                              — replay a past
+                                                        execution trace
 
 Run from container:
   podman run --rm --env-file .env.local \\
@@ -12,23 +17,33 @@
 
 Or directly:
   PYTHONPATH=src python -m forge.cli review src/forge/main.py
+  PYTHONPATH=src python -m forge.cli review src/forge/graph.py --tests tests/test_graph.py
 """
 
 import sys
 
 
 def _cmd_review(args: list[str]) -> int:
     if not args:
-        print("Usage: forge review <file> [question]", file=sys.stderr)
+        print("Usage: forge review <file> [question] [--tests <path>]", file=sys.stderr)
         return 1
 
+    test_path = None
+    if "--tests" in args:
+        idx = args.index("--tests")
+        if idx + 1 >= len(args):
+            print("Usage: --tests requires a path argument", file=sys.stderr)
+            return 1
+        test_path = args[idx + 1]
+        args = args[:idx] + args[idx + 2 :]
+
     file_path = args[0]
     question = " ".join(args[1:]) if len(args) > 1 else "Que peut-on améliorer ?"
 
     from forge.graphs.review import run
 
     print(f"Reviewing {file_path!r}…\n")
-    result = run(file_path, question=question)
+    result = run(file_path, question=question, test_path=test_path)
     print(result)
     return 0