feat: add built-in fetch skill (read webpage text)

README.md

@@ -4,7 +4,7 @@ A minimal CLI coding agent with a persistent IPython execution environment and o
 
 The model gets a single built-in tool, `ipython`: a persistent IPython kernel for Python, shell commands via `!command`, and multi-line shell scripts via `%%bash`. The tool set is not configurable. File edits, shell work, and orchestration all go through it.
 
-For convenience, rlm ships built-in *skills* that can be enabled per run via `RLM_SKILLS` (comma-separated, off by default): `edit` (single-occurrence string replacement) and `search` (web search via Serper, needs `SERPER_API_KEY`). Enabled skills are pre-imported into the IPython kernel like any other skill (see [Skills](#skills)), so the agent calls `await edit(path=..., old_str=..., new_str=...)` or `await search(query=...)`.
+For convenience, rlm ships built-in *skills* that can be enabled per run via `RLM_SKILLS` (comma-separated, off by default): `edit` (single-occurrence string replacement), `search` (web search via Serper, needs `SERPER_API_KEY`), and `fetch` (retrieve a URL as cleaned text). Enabled skills are pre-imported into the IPython kernel like any other skill (see [Skills](#skills)), so the agent calls `await edit(path=..., old_str=..., new_str=...)`, `await search(query=...)`, or `await fetch(url=...)`.
 
 Context is reclaimed automatically: when a turn's prompt token count crosses `RLM_SUMMARIZE_AT_TOKENS`, the engine compacts the conversation into a summary and continues on a fresh branch. The IPython kernel keeps running across the compaction, so REPL state survives (see [Compaction](#compaction)).
 
@@ -99,7 +99,7 @@ versioned contract described above.
 | `SERPER_API_KEY` | — | API key for the built-in `search` skill (Serper backend). Resolved by the supervisor and not copied into the kernel. |
 | `PRIME_API_KEY` | — | PI Inference pair: targets `https://api.pinference.ai/api/v1` and forwards `PRIME_TEAM_ID` as `X-Prime-Team-ID` when set. |
 | `OPENAI_API_KEY` / `OPENAI_BASE_URL` | resolved at startup | OpenAI pair (covers OpenAI direct and verifiers' rollout tunnel). Provider precedence: explicit → PI → OpenAI. Keys are scoped to their own base URL so an `OPENAI_API_KEY` lying around can't leak to PI Inference. |
-| `RLM_SKILLS` | — | Comma-separated built-in skills to enable (`edit`, `search`); pre-imported into the kernel. Unknown names raise. See [Skills](#skills). |
+| `RLM_SKILLS` | — | Comma-separated built-in skills to enable (`edit`, `search`, `fetch`); pre-imported into the kernel. Unknown names raise. See [Skills](#skills). |
 | `RLM_MCP_CONFIG` | — | Standard `mcpServers` config (streamable HTTP or stdio); each server's tools become pre-imported IPython skills (`<server>_<tool>`). See [MCP tools as skills](#mcp-tools-as-skills). |
 | `RLM_KERNEL_ENV` | `{}` | JSON object of task variables explicitly passed to IPython and its subprocesses. Supervisor, provider, MCP, and broker configuration names are reserved. |
 | `RLM_MAX_DEPTH` | `0` | Max recursion depth (`0` means no sub-agents) |

src/rlm/prompt.py

@@ -55,6 +55,12 @@
     "several angles at once, fan out with `asyncio.gather(search(...), search(...))`."
 )
 
+FETCH_SKILL_PROMPT = (
+    "To read a specific webpage, use the pre-imported async `fetch` skill: "
+    '`await fetch(url="...")` returns the webpage as cleaned text. Use it to open URLs '
+    "from `search` results."
+)
+
 
 def build_system_prompt(
     cwd: str,
@@ -137,6 +143,8 @@ def build_system_prompt(
             skill_lines.append(EDIT_SKILL_PROMPT)
         if "search" in installed_skills:
             skill_lines.append(SEARCH_SKILL_PROMPT)
+        if "fetch" in installed_skills:
+            skill_lines.append(FETCH_SKILL_PROMPT)
     if skill_lines:
         parts.extend(["", *skill_lines])
 

src/rlm/skills/__init__.py

@@ -14,6 +14,7 @@
 _BUILTIN_SKILLS: dict[str, str | None] = {
     "bash": "rlm.skills.bash",
     "edit": "rlm.skills.edit",
+    "fetch": "rlm.skills.fetch",
     "search": None,
 }
 

src/rlm/skills/fetch.py

@@ -0,0 +1,59 @@
+"""Built-in ``fetch`` skill — retrieve a webpage and return its cleaned text.
+
+Enabled via ``RLM_SKILLS``; pre-imported into the IPython kernel so the agent calls
+``await fetch(url="...")``. Gives the model a proper tool for reading a webpage instead of
+hand-rolling ``curl``/``requests``/``urllib`` (which tend to return raw HTML, spam, or errors).
+Pairs with the ``search`` skill: ``search`` finds URLs, ``fetch`` reads them.
+"""
+
+from __future__ import annotations
+
+import html as _html
+import re
+
+import httpx
+
+DEFAULT_MAX_CHARS = 20_000
+
+_TAG_BLOCKS = re.compile(r"(?is)<(script|style|noscript|template|svg)[^>]*>.*?</\1>")
+_TAGS = re.compile(r"(?s)<[^>]+>")
+_WS = re.compile(r"\s+")
+
+
+def html_to_text(body: str) -> str:
+    """Strip scripts/styles/tags and collapse whitespace into readable text."""
+    body = _TAG_BLOCKS.sub(" ", body)
+    body = _TAGS.sub(" ", body)
+    return _WS.sub(" ", _html.unescape(body)).strip()
+
+
+async def run(url: str, *, max_chars: int = DEFAULT_MAX_CHARS) -> str:
+    """Fetch a webpage and return its cleaned text content (truncated to ``max_chars``).
+
+    Args:
+        url: The webpage to fetch (``https://`` assumed if no scheme).
+        max_chars: Truncate the returned text to this many characters.
+
+    Returns:
+        ``URL: <url>`` followed by the webpage's cleaned text, or a short error string.
+    """
+    if not re.match(r"^https?://", url):
+        url = "https://" + url
+    try:
+        async with httpx.AsyncClient(follow_redirects=True, timeout=45) as client:
+            response = await client.get(
+                url, headers={"User-Agent": "Mozilla/5.0 (compatible; rlm-fetch)"}
+            )
+        response.raise_for_status()
+    except Exception as exc:
+        return f"Error fetching {url}: {type(exc).__name__}: {exc}"
+    content_type = response.headers.get("content-type", "").lower()
+    try:
+        body = response.text
+    except (UnicodeDecodeError, LookupError):
+        body = response.content.decode("utf-8", errors="replace")
+    is_html = "html" in content_type or "<html" in body[:1000].lower()
+    text = html_to_text(body) if is_html else body.strip()
+    if len(text) > max_chars:
+        text = text[:max_chars] + f"\n... [truncated at {max_chars} chars]"
+    return f"URL: {url}\n\n{text}"

tests/test_builtin_skills.py

@@ -199,3 +199,22 @@ def test_enable_bash_skill_writes_stub(tmp_path):
 async def test_bash_survives_binary_output():
     out = await bash("printf 'head\\xff\\xfetail'")
     assert "head" in out and "tail" in out
+
+
+def test_enable_fetch_skill_writes_stub(tmp_path):
+    enabled = enable_builtin_skills(["fetch"], tmp_path)
+    assert enabled == ["fetch"]
+    stub = (tmp_path / "fetch.py").read_text()
+    assert "from rlm.skills.fetch import run" in stub
+
+
+def test_fetch_html_to_text_strips_markup():
+    from rlm.skills.fetch import html_to_text
+
+    html = (
+        "<html><head><style>x{}</style></head><body><h1>T</h1>"
+        "<p>Hello &amp; world</p><script>bad()</script><p>Line2</p></body></html>"
+    )
+    out = html_to_text(html)
+    assert "Hello & world" in out and "Line2" in out
+    assert "bad()" not in out and "<" not in out