feat: add bash context compaction

tests/v1/test_context.py

@@ -0,0 +1,70 @@
+import httpx
+from openai import BadRequestError
+
+from verifiers.v1.clients.context import (
+    compaction_threshold,
+    model_context_window,
+)
+from verifiers.v1.configs.harness import CompactionConfig
+from verifiers.v1.harnesses.bash.harness import BashHarnessConfig
+from verifiers.v1.harnesses.bash.program import context_error
+from verifiers.v1.harnesses.rlm.harness import RLMHarnessConfig
+
+
+def test_model_context_window_reads_vllm_extension() -> None:
+    payload = {
+        "data": [
+            {"id": "other", "max_model_len": 1},
+            {"id": "target", "max_model_len": 32_768},
+        ]
+    }
+
+    assert model_context_window(payload, "target") == 32_768
+
+
+def test_model_context_window_accepts_common_provider_extensions() -> None:
+    payload = {"data": [{"id": "target", "context_length": 128_000}]}
+
+    assert model_context_window(payload, "target") == 128_000
+
+
+def test_model_context_window_is_unknown_for_standard_model_card() -> None:
+    payload = {
+        "data": [
+            {
+                "id": "target",
+                "object": "model",
+                "created": 1,
+                "owned_by": "provider",
+            }
+        ]
+    }
+
+    assert model_context_window(payload, "target") is None
+
+
+def test_compaction_threshold_reserves_ten_percent() -> None:
+    assert compaction_threshold(32_768) == 29_491
+
+
+def test_compaction_is_disabled_by_default_for_both_harnesses() -> None:
+    assert BashHarnessConfig().compaction is None
+    assert RLMHarnessConfig().compaction is None
+
+
+def test_compaction_config_has_shared_automatic_default() -> None:
+    assert CompactionConfig().summarize_at_tokens is None
+
+
+def test_threshold_is_learned_from_provider_error() -> None:
+    response = httpx.Response(
+        400,
+        request=httpx.Request("POST", "http://provider/v1/chat/completions"),
+    )
+    error = BadRequestError(
+        "maximum context length is 32,768 tokens",
+        response=response,
+        body={"error": {"message": "maximum context length is 32,768 tokens"}},
+    )
+
+    assert context_error(error) == (True, 29_491)

verifiers/v1/clients/context.py

@@ -0,0 +1,50 @@
+"""Model context-window discovery for OpenAI-compatible endpoints."""
+
+from collections.abc import Mapping
+from typing import Any, cast
+
+from openai import APIError
+
+from verifiers.v1.clients.base import build_async_openai
+from verifiers.v1.clients.client import ModelContext
+
+CONTEXT_WINDOW_FIELDS = (
+    "max_model_len",
+    "context_length",
+    "context_window",
+    "max_context_length",
+)
+_context_window_cache: dict[tuple[str, str], int | None] = {}
+
+
+def model_context_window(payload: Mapping[str, Any], model: str) -> int | None:
+    """Read a provider context-window extension from one model card."""
+    for card in payload.get("data") or []:
+        if not isinstance(card, Mapping) or card.get("id") != model:
+            continue
+        for field in CONTEXT_WINDOW_FIELDS:
+            value = card.get(field)
+            if isinstance(value, int) and not isinstance(value, bool) and value > 0:
+                return value
+        break
+    return None
+
+
+def compaction_threshold(context_window: int) -> int:
+    """Reserve ten percent of the model context for checkpointing."""
+    return max(1, context_window * 9 // 10)
+
+
+async def resolve_compaction_threshold(ctx: ModelContext) -> int | None:
+    """Discover a model's proactive compaction threshold when advertised."""
+    key = (ctx.client.model_dump_json(), ctx.model)
+    if key not in _context_window_cache:
+        try:
+            async with build_async_openai(ctx.client) as client:
+                payload = await client.get("/models", cast_to=cast(Any, dict[str, Any]))
+            _context_window_cache[key] = model_context_window(payload, ctx.model)
+        except APIError:
+            _context_window_cache[key] = None
+
+    window = _context_window_cache[key]
+    return compaction_threshold(window) if window is not None else None

verifiers/v1/configs/harness.py

@@ -3,14 +3,39 @@
 from __future__ import annotations
 
 import os
+import random
 from pathlib import Path
 
-from pydantic import ConfigDict, Field, FiniteFloat
+from pydantic import ConfigDict, Field, FiniteFloat, PositiveInt, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.types import ID
 
 
+class CompactionConfig(BaseConfig):
+    """Optional context compaction policy for in-house agent loops."""
+
+    summarize_at_tokens: PositiveInt | tuple[PositiveInt, PositiveInt] | None = None
+    """Compact at this token count. A pair draws a task-seeded threshold. When unset, use
+    90% of the model context window when the provider advertises it."""
+
+    @model_validator(mode="after")
+    def validate_range(self) -> CompactionConfig:
+        value = self.summarize_at_tokens
+        if isinstance(value, tuple) and value[0] > value[1]:
+            raise ValueError(
+                "`summarize_at_tokens` range must be (lo, hi) with lo <= hi."
+            )
+        return self
+
+    def summarize_threshold(self, task_idx: int | None) -> int | None:
+        value = self.summarize_at_tokens
+        if isinstance(value, tuple):
+            lo, hi = value
+            return random.Random(task_idx or 0).randint(lo, hi)
+        return value
+
+
 class HarnessConfig(BaseConfig):
     id: ID = "bash"
     """Installed harness package, set through the seat's

verifiers/v1/harnesses/bash/harness.py

@@ -3,7 +3,8 @@
 from pathlib import Path
 
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.clients.context import resolve_compaction_threshold
+from verifiers.v1.configs.harness import CompactionConfig, HarnessConfig
 from verifiers.v1.dialects.chat import message_to_wire
 from verifiers.v1.harness import Harness
 from verifiers.v1.runtimes import ProgramResult, Runtime
@@ -28,6 +29,9 @@
 
 
 class BashHarnessConfig(HarnessConfig):
+    compaction: CompactionConfig | None = None
+    """Context compaction policy. Set an empty config to use automatic thresholds."""
+
     edit: bool = True
     """Offer the local `edit` tool (single-occurrence string replacement in a file) alongside
     `bash`. On by default; set `--env.agent.harness.edit false` for a bash-only agent."""
@@ -77,6 +81,13 @@ async def launch(
         ]
         if tool_interception_url:
             args.append(f"--tool-interception-url={tool_interception_url}")
+        if self.config.compaction is not None:
+            args.append("--compaction")
+            threshold = self.config.compaction.summarize_threshold(data.idx)
+            if threshold is None:
+                threshold = await resolve_compaction_threshold(ctx)
+            if threshold is not None:
+                args.append(f"--summarize-at-tokens={threshold}")
         if self.config.edit:
             args.append("--edit")
         if self.config.search: