diff --git a/README.md b/README.md index cf18aba..c0c9089 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A minimal CLI coding agent with a persistent IPython execution environment and o The model gets a single built-in tool, `ipython`: a persistent IPython kernel for Python, shell commands via `!command`, and multi-line shell scripts via `%%bash`. The tool set is not configurable. File edits, shell work, and orchestration all go through it. -For convenience, rlm ships built-in *skills* that can be enabled per run via `RLM_SKILLS` (comma-separated, off by default): `edit` (single-occurrence string replacement) and `search` (web search via Serper, needs `SERPER_API_KEY`). Enabled skills are pre-imported into the IPython kernel like any other skill (see [Skills](#skills)), so the agent calls `await edit(path=..., old_str=..., new_str=...)` or `await search(query=...)`. +For convenience, rlm ships built-in *skills* that can be enabled per run via `RLM_SKILLS` (comma-separated, off by default): `edit` (single-occurrence string replacement), `search` (web search via Serper, needs `SERPER_API_KEY`), and `fetch` (retrieve a URL as cleaned text). Enabled skills are pre-imported into the IPython kernel like any other skill (see [Skills](#skills)), so the agent calls `await edit(path=..., old_str=..., new_str=...)`, `await search(query=...)`, or `await fetch(url=...)`. Context is reclaimed automatically: when a turn's prompt token count crosses `RLM_SUMMARIZE_AT_TOKENS`, the engine compacts the conversation into a summary and continues on a fresh branch. The IPython kernel keeps running across the compaction, so REPL state survives (see [Compaction](#compaction)). @@ -99,7 +99,7 @@ versioned contract described above. | `SERPER_API_KEY` | — | API key for the built-in `search` skill (Serper backend). Resolved by the supervisor and not copied into the kernel. | | `PRIME_API_KEY` | — | PI Inference pair: targets `https://api.pinference.ai/api/v1` and forwards `PRIME_TEAM_ID` as `X-Prime-Team-ID` when set. | | `OPENAI_API_KEY` / `OPENAI_BASE_URL` | resolved at startup | OpenAI pair (covers OpenAI direct and verifiers' rollout tunnel). Provider precedence: explicit → PI → OpenAI. Keys are scoped to their own base URL so an `OPENAI_API_KEY` lying around can't leak to PI Inference. | -| `RLM_SKILLS` | — | Comma-separated built-in skills to enable (`edit`, `search`); pre-imported into the kernel. Unknown names raise. See [Skills](#skills). | +| `RLM_SKILLS` | — | Comma-separated built-in skills to enable (`edit`, `search`, `fetch`); pre-imported into the kernel. Unknown names raise. See [Skills](#skills). | | `RLM_MCP_CONFIG` | — | Standard `mcpServers` config (streamable HTTP or stdio); each server's tools become pre-imported IPython skills (`_`). See [MCP tools as skills](#mcp-tools-as-skills). | | `RLM_KERNEL_ENV` | `{}` | JSON object of task variables explicitly passed to IPython and its subprocesses. Supervisor, provider, MCP, and broker configuration names are reserved. | | `RLM_MAX_DEPTH` | `0` | Max recursion depth (`0` means no sub-agents) | diff --git a/src/rlm/prompt.py b/src/rlm/prompt.py index 6d42356..af34c14 100644 --- a/src/rlm/prompt.py +++ b/src/rlm/prompt.py @@ -55,6 +55,12 @@ "several angles at once, fan out with `asyncio.gather(search(...), search(...))`." ) +FETCH_SKILL_PROMPT = ( + "To read a specific webpage, use the pre-imported async `fetch` skill: " + '`await fetch(url="...")` returns the webpage as cleaned text. Use it to open URLs ' + "from `search` results." +) + def build_system_prompt( cwd: str, @@ -137,6 +143,8 @@ def build_system_prompt( skill_lines.append(EDIT_SKILL_PROMPT) if "search" in installed_skills: skill_lines.append(SEARCH_SKILL_PROMPT) + if "fetch" in installed_skills: + skill_lines.append(FETCH_SKILL_PROMPT) if skill_lines: parts.extend(["", *skill_lines]) diff --git a/src/rlm/skills/__init__.py b/src/rlm/skills/__init__.py index 849523b..5d2f3a7 100644 --- a/src/rlm/skills/__init__.py +++ b/src/rlm/skills/__init__.py @@ -14,6 +14,7 @@ _BUILTIN_SKILLS: dict[str, str | None] = { "bash": "rlm.skills.bash", "edit": "rlm.skills.edit", + "fetch": "rlm.skills.fetch", "search": None, } diff --git a/src/rlm/skills/fetch.py b/src/rlm/skills/fetch.py new file mode 100644 index 0000000..eead7c1 --- /dev/null +++ b/src/rlm/skills/fetch.py @@ -0,0 +1,59 @@ +"""Built-in ``fetch`` skill — retrieve a webpage and return its cleaned text. + +Enabled via ``RLM_SKILLS``; pre-imported into the IPython kernel so the agent calls +``await fetch(url="...")``. Gives the model a proper tool for reading a webpage instead of +hand-rolling ``curl``/``requests``/``urllib`` (which tend to return raw HTML, spam, or errors). +Pairs with the ``search`` skill: ``search`` finds URLs, ``fetch`` reads them. +""" + +from __future__ import annotations + +import html as _html +import re + +import httpx + +DEFAULT_MAX_CHARS = 20_000 + +_TAG_BLOCKS = re.compile(r"(?is)<(script|style|noscript|template|svg)[^>]*>.*?") +_TAGS = re.compile(r"(?s)<[^>]+>") +_WS = re.compile(r"\s+") + + +def html_to_text(body: str) -> str: + """Strip scripts/styles/tags and collapse whitespace into readable text.""" + body = _TAG_BLOCKS.sub(" ", body) + body = _TAGS.sub(" ", body) + return _WS.sub(" ", _html.unescape(body)).strip() + + +async def run(url: str, *, max_chars: int = DEFAULT_MAX_CHARS) -> str: + """Fetch a webpage and return its cleaned text content (truncated to ``max_chars``). + + Args: + url: The webpage to fetch (``https://`` assumed if no scheme). + max_chars: Truncate the returned text to this many characters. + + Returns: + ``URL: `` followed by the webpage's cleaned text, or a short error string. + """ + if not re.match(r"^https?://", url): + url = "https://" + url + try: + async with httpx.AsyncClient(follow_redirects=True, timeout=45) as client: + response = await client.get( + url, headers={"User-Agent": "Mozilla/5.0 (compatible; rlm-fetch)"} + ) + response.raise_for_status() + except Exception as exc: + return f"Error fetching {url}: {type(exc).__name__}: {exc}" + content_type = response.headers.get("content-type", "").lower() + try: + body = response.text + except (UnicodeDecodeError, LookupError): + body = response.content.decode("utf-8", errors="replace") + is_html = "html" in content_type or " max_chars: + text = text[:max_chars] + f"\n... [truncated at {max_chars} chars]" + return f"URL: {url}\n\n{text}" diff --git a/tests/test_builtin_skills.py b/tests/test_builtin_skills.py index fa2d7f4..e1f39cc 100644 --- a/tests/test_builtin_skills.py +++ b/tests/test_builtin_skills.py @@ -199,3 +199,22 @@ def test_enable_bash_skill_writes_stub(tmp_path): async def test_bash_survives_binary_output(): out = await bash("printf 'head\\xff\\xfetail'") assert "head" in out and "tail" in out + + +def test_enable_fetch_skill_writes_stub(tmp_path): + enabled = enable_builtin_skills(["fetch"], tmp_path) + assert enabled == ["fetch"] + stub = (tmp_path / "fetch.py").read_text() + assert "from rlm.skills.fetch import run" in stub + + +def test_fetch_html_to_text_strips_markup(): + from rlm.skills.fetch import html_to_text + + html = ( + "

T

" + "

Hello & world

Line2

" + ) + out = html_to_text(html) + assert "Hello & world" in out and "Line2" in out + assert "bad()" not in out and "<" not in out