From 99e432231651739131a512bbb4d66252d298eb42 Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Sun, 23 Aug 2026 18:08:54 -0700 Subject: [PATCH 1/8] feat: add built-in Cline harness --- verifiers/v1/harnesses/__init__.py | 3 + verifiers/v1/harnesses/cline/__init__.py | 3 + verifiers/v1/harnesses/cline/harness.py | 200 +++++++++++++++++++++++ 3 files changed, 206 insertions(+) create mode 100644 verifiers/v1/harnesses/cline/__init__.py create mode 100644 verifiers/v1/harnesses/cline/harness.py diff --git a/verifiers/v1/harnesses/__init__.py b/verifiers/v1/harnesses/__init__.py index 3638525183..802c39d78b 100644 --- a/verifiers/v1/harnesses/__init__.py +++ b/verifiers/v1/harnesses/__init__.py @@ -7,6 +7,7 @@ ClaudeCodeHarness, ClaudeCodeHarnessConfig, ) +from verifiers.v1.harnesses.cline import ClineHarness, ClineHarnessConfig from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig from verifiers.v1.harnesses.hermes_agent import ( HermesAgentHarness, @@ -31,6 +32,8 @@ "BrowserUseHarnessConfig", "ClaudeCodeHarness", "ClaudeCodeHarnessConfig", + "ClineHarness", + "ClineHarnessConfig", "CodexHarness", "CodexHarnessConfig", "HermesAgentHarness", diff --git a/verifiers/v1/harnesses/cline/__init__.py b/verifiers/v1/harnesses/cline/__init__.py new file mode 100644 index 0000000000..18ee30ca04 --- /dev/null +++ b/verifiers/v1/harnesses/cline/__init__.py @@ -0,0 +1,3 @@ +from verifiers.v1.harnesses.cline.harness import ClineHarness, ClineHarnessConfig + +__all__ = ["ClineHarness", "ClineHarnessConfig"] diff --git a/verifiers/v1/harnesses/cline/harness.py b/verifiers/v1/harnesses/cline/harness.py new file mode 100644 index 0000000000..6ed08c4e98 --- /dev/null +++ b/verifiers/v1/harnesses/cline/harness.py @@ -0,0 +1,200 @@ +"""Run the public Cline CLI headlessly through interception.""" + +import json +import logging +import shlex +from typing import Literal + +from pydantic import Field + +from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.harness import HarnessConfig +from verifiers.v1.harness import Harness +from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node +from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace + +logger = logging.getLogger(__name__) + +CLINE_DIR = "/var/tmp/vf-cline-{version}" +PACKAGES_DIR = f"{CLINE_DIR}/packages" +# Cline's package publishes its own platform resolver. npm 11 does not create a +# `.bin/cline` shim for this package in the minimal prefix install, so address the +# declared binary directly. +CLINE_BIN = f"{PACKAGES_DIR}/node_modules/cline/bin/cline" +CLINE_DATA_ROOT = "/tmp/vf-cline" + +# Keep the initial coding harness local to the task runtime. These tools add +# external information, human interaction, or nested agents whose work would not +# be represented as the primary harness's ordinary tool loop. +RESTRICTED_TOOLS = ( + "fetch_web_content", + "skills", + "ask_question", + "spawn_agent", + "team_spawn_teammate", + "team_shutdown_teammate", + "team_status", + "team_task", + "team_run_task", + "team_cancel_run", + "team_list_runs", + "team_await_runs", + "team_send_message", + "team_broadcast", + "team_read_mailbox", + "team_mission_log", + "team_cleanup", + "team_create_outcome", + "team_attach_outcome_fragment", + "team_review_outcome_fragment", + "team_finalize_outcome", + "team_list_outcomes", +) + +INSTALL = r""" +set -e +export PATH="/var/tmp/vf-node/bin:$PATH" +rm -f {ready} +npm install --prefix {packages} --no-audit --no-fund --omit=dev \ + "cline@$VF_CLINE_VERSION" >/dev/null +touch {ready} +""" + + +class ClineHarnessConfig(HarnessConfig): + version: str = Field(default="3.0.57", pattern=r"^[A-Za-z0-9._+-]+$") + """Public Cline CLI release to install, pinned for reproducibility.""" + + compaction: Literal["agentic", "basic", "off"] = "basic" + """Cline's context-compaction mode.""" + + max_retries: int = Field(default=6, ge=1) + """Maximum consecutive Cline mistakes before the CLI exits.""" + + +class ClineHarness(Harness[ClineHarnessConfig]): + async def setup(self, runtime: Runtime) -> None: + await ensure_node(runtime) + directory = CLINE_DIR.format(version=self.config.version) + packages = PACKAGES_DIR.format(version=self.config.version) + cline_bin = CLINE_BIN.format(version=self.config.version) + ready = f"{directory}/.ready" + script = INSTALL.replace("{packages}", packages).replace("{ready}", ready) + ensure = shlex.quote(f"[ -f {ready} ] && [ -x {cline_bin} ] || ({script})") + guarded = ( + f"mkdir -p {directory} && " + f'"$(command -v flock || command -v lockf)" {directory}/install.lock ' + f"sh -c {ensure}" + ) + logger.info("cline: ensuring Cline CLI %s is installed", self.config.version) + result = await runtime.run( + ["sh", "-c", guarded], + { + **self.config.resolved_env, + "VF_CLINE_VERSION": self.config.version, + }, + ) + if result.exit_code != 0: + detail = (result.stderr or result.stdout).strip()[-500:] + raise RuntimeError(f"Cline CLI install failed: {detail}") + + async def launch( + self, + ctx: ModelContext, + trace: Trace, + runtime: Runtime, + endpoint: str, + secret: str, + mcp_urls: dict[str, str], + data: TaskData, + ) -> ProgramResult: + if mcp_urls: + raise ValueError("Cline harness v1 does not support MCP servers") + _, prompt = self.resolve_text_prompt(data) + if prompt is None: + raise ValueError("Cline requires a task prompt") + + data_dir = self.data_dir(trace) + settings_dir = f"{data_dir}/settings" + mcp_settings = f"{settings_dir}/cline_mcp_settings.json" + disabled_tools = list( + dict.fromkeys([*RESTRICTED_TOOLS, *(self.config.disabled_tools or [])]) + ) + await runtime.write( + f"{settings_dir}/global-settings.json", + json.dumps({"disabledTools": disabled_tools}).encode(), + ) + await runtime.write(mcp_settings, b'{"mcpServers":{}}') + + env = { + **self.config.resolved_env, + "PATH": ( + f"{NODE_BIN_DIR}:/usr/local/sbin:/usr/local/bin:" + "/usr/sbin:/usr/bin:/sbin:/bin" + ), + "CLINE_DATA_DIR": data_dir, + "CLINE_MCP_SETTINGS_PATH": mcp_settings, + "CLINE_TELEMETRY_DISABLED": "1", + "CLINE_NO_AUTO_UPDATE": "1", + "NO_UPDATE_NOTIFIER": "1", + } + # Execute npm's launcher through its shebang. Passing the `.bin` symlink + # to `node` directly bypasses the package launcher's intended resolution + # path for the platform-specific compiled binary. + cline = [CLINE_BIN.format(version=self.config.version)] + auth = await runtime.run( + [ + *cline, + "auth", + "--provider", + "openai-compatible", + "--apikey", + secret, + "--modelid", + ctx.model, + "--baseurl", + endpoint, + "--data-dir", + data_dir, + ], + env, + ) + if auth.exit_code != 0: + detail = (auth.stderr or auth.stdout).strip()[-500:] + raise RuntimeError(f"Cline provider configuration failed: {detail}") + + args = [ + *cline, + "--json", + "--auto-approve", + "true", + "--cwd", + ".", + "--provider", + "openai-compatible", + "--key", + secret, + "--model", + ctx.model, + "--compaction", + self.config.compaction, + "--retries", + str(self.config.max_retries), + "--data-dir", + data_dir, + ] + if effort := ctx.sampling.reasoning_effort: + args += ["--thinking", effort] + return await runtime.run_program([*args, prompt], env) + + async def cleanup(self, trace: Trace, runtime: Runtime) -> None: + result = await runtime.run(["rm", "-rf", self.data_dir(trace)], {}) + if result.exit_code != 0: + detail = (result.stderr or result.stdout).strip()[-500:] + raise RuntimeError(f"failed to clean up Cline data: {detail}") + + @staticmethod + def data_dir(trace: Trace) -> str: + return f"{CLINE_DATA_ROOT}/{trace.id}" From 0dace07e703b1c501714fcb4f24f022263eddabb Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Sun, 23 Aug 2026 18:08:59 -0700 Subject: [PATCH 2/8] test: cover Cline in agentic e2e matrix --- pyproject.toml | 1 + tests/v1/conftest.py | 6 +++--- tests/v1/test_e2e.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aa60eeda04..88456761f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -293,6 +293,7 @@ markers = [ "kimi_code: v1 e2e cases on the kimi-code harness", "pi: v1 e2e cases on the pi harness", "pool: v1 e2e cases on the pool harness", + "cline: v1 e2e cases on the Cline harness", "codex: v1 e2e cases on the codex harness", "unit: marks tests as unit tests", "asyncio: marks tests as async tests", diff --git a/tests/v1/conftest.py b/tests/v1/conftest.py index 0b5ba36de9..548c2a21e0 100644 --- a/tests/v1/conftest.py +++ b/tests/v1/conftest.py @@ -24,8 +24,8 @@ uv run pytest tests/v1 -n auto -m modal # only modal (needs local setup) Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`, -harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `openclaw` / `codex` / -`claude_code` / `hermes_agent`. +harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `openclaw` / `cline` / +`codex` / `claude_code` / `hermes_agent`. A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only. @@ -71,7 +71,7 @@ def tool_runtime(request) -> dict: # Built-in harnesses are bundled in the `harnesses` package; the agent CLIs (`rlm` / -# `kimi-code` / `openclaw` / `codex` / `claude-code` / `hermes-agent`) install their +# `kimi-code` / `openclaw` / `cline` / `codex` / `claude-code` / `hermes-agent`) install their # dependencies at rollout. # `compact` (an example harness) and `terminus-2` (drives the host tmux) are excluded from e2e. @pytest.fixture diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index 500f4112fc..c2a816c089 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -45,6 +45,7 @@ def pair(a: str, b: str, id: str, *extra_marks): marks=[mark.kimi_code, mark.docker], id="kimi-code-responses-harness-in-docker", ), + pair("cline", "docker", "cline-harness-in-docker"), pair("codex", "docker", "codex-harness-in-docker"), pair("claude-code", "docker", "claude-code-harness-in-docker"), pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"), From 58f6efc28b7154af82a6af128dd76fdd8f5369f9 Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Sun, 23 Aug 2026 18:09:00 -0700 Subject: [PATCH 3/8] docs: document Cline harness usage and limits --- verifiers/v1/harnesses/cline/README.md | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 verifiers/v1/harnesses/cline/README.md diff --git a/verifiers/v1/harnesses/cline/README.md b/verifiers/v1/harnesses/cline/README.md new file mode 100644 index 0000000000..1f628c6c3c --- /dev/null +++ b/verifiers/v1/harnesses/cline/README.md @@ -0,0 +1,71 @@ +# Cline harness + +The `cline` harness runs the public Cline CLI inside the selected verifiers +runtime and routes every model call through the rollout's interception server. +Cline's native OpenAI-compatible tool calls, results, usage, and final response +therefore become ordinary verifiers trace nodes and model-call records. + +## Quickstart + +Choose an isolated container or VM runtime. The harness installs its pinned Cline +release during trusted setup. + +```bash +export OPENAI_API_KEY=... +uv run eval MY_TASKSET \ + --env.agent.harness.id cline \ + --env.agent.runtime.type docker \ + --model openai/gpt-4.1-mini \ + --client.base-url https://openrouter.ai/api/v1 \ + --client.api-key-var OPENAI_API_KEY \ + --no-push +``` + +`--no-push` keeps the run local. Inspect the resolved config and traces under +`outputs//configs/eval.json` and `outputs//traces.jsonl`. + +## Harness configuration + +```toml +[env.agent.harness] +id = "cline" +version = "3.0.57" +compaction = "basic" # basic | agentic | off +max_retries = 6 +disabled_tools = [] # additional native Cline tool names +``` + +Use the agent and sampling blocks for settings owned by verifiers: + +```toml +[env.agent] +max_turns = 20 +max_output_tokens = 32768 + +[env.agent.timeout] +setup = 600 +rollout = 1800 + +[sampling] +temperature = 0.2 +``` + +The interception server applies the configured model and sampling values to +Cline's requests before forwarding them upstream. + +## Security posture and limitations + +- Each rollout gets an isolated Cline data directory, deleted after scoring. +- Cline telemetry and update checks are disabled. +- Cline MCP configuration starts empty. The harness does not advertise MCP or + resume support yet. +- Network/human/nested-agent tools are disabled by default. The retained tools + read, search, edit, and run commands in the task runtime. +- The adapter uses Cline's one-shot headless mode. Multi-turn tool use inside one + task is captured, but env-driven user simulation requires future ACP support. +- Cline `3.0.57` has no max-step flag. Use verifiers' `max_turns`, token caps, and + rollout timeout; their stop conditions and errors are recorded on the trace. +- Eval-client traces preserve message/tool structure and provider-reported usage, + but exact token IDs and masks require a compatible training client/renderer. +- Compaction was not forced in the smoke validation. If Cline rewrites history, + inspect the resulting branches before using those samples for training. From 87dcc418bc5d2c5ed64ec403076cbc19a66fab98 Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Sun, 23 Aug 2026 18:19:05 -0700 Subject: [PATCH 4/8] test: verify Cline harness launch plumbing --- tests/v1/test_cline_harness.py | 110 +++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 tests/v1/test_cline_harness.py diff --git a/tests/v1/test_cline_harness.py b/tests/v1/test_cline_harness.py new file mode 100644 index 0000000000..5295475953 --- /dev/null +++ b/tests/v1/test_cline_harness.py @@ -0,0 +1,110 @@ +from typing import cast + +import pytest +from pydantic import ValidationError + +from verifiers.v1.clients import ModelContext +from verifiers.v1.configs.client import EvalClientConfig +from verifiers.v1.harnesses.cline import ClineHarness, ClineHarnessConfig +from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace +from verifiers.v1.types import Sampling + + +class RecordingRuntime: + def __init__(self) -> None: + self.commands: list[tuple[list[str], dict[str, str]]] = [] + self.programs: list[tuple[list[str], dict[str, str]]] = [] + self.files: dict[str, bytes] = {} + + async def write(self, path: str, data: bytes) -> None: + self.files[path] = data + + async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult: + self.commands.append((argv, env)) + return ProgramResult(exit_code=0, stdout="", stderr="") + + async def run_program(self, argv: list[str], env: dict[str, str]) -> ProgramResult: + self.programs.append((argv, env)) + return ProgramResult(exit_code=0, stdout="", stderr="") + + +def test_cline_config_validates_pinned_version_and_retries() -> None: + config = ClineHarnessConfig(id="cline") + assert config.version == "3.0.57" + assert config.compaction == "basic" + assert config.max_retries == 6 + + with pytest.raises(ValidationError): + ClineHarnessConfig(id="cline", version="3.0.57; unsafe") + with pytest.raises(ValidationError): + ClineHarnessConfig(id="cline", max_retries=0) + + +@pytest.mark.asyncio +async def test_cline_launch_wires_interception_and_restricts_tools() -> None: + runtime = RecordingRuntime() + harness = ClineHarness( + ClineHarnessConfig(id="cline", disabled_tools=["custom_external_tool"]) + ) + ctx = ModelContext( + model="openai/example", + client=EvalClientConfig(), + sampling=Sampling(reasoning_effort="medium"), + ) + trace = Trace.model_construct(id="trace-id") + + result = await harness.launch( + ctx, + trace, + cast(Runtime, runtime), + "http://interception.test/v1", + "ephemeral-secret", + {}, + TaskData(prompt="solve the task", system_prompt="follow the rules"), + ) + + assert result.exit_code == 0 + settings = runtime.files[ + "/tmp/vf-cline/trace-id/settings/global-settings.json" + ].decode() + assert '"fetch_web_content"' in settings + assert '"spawn_agent"' in settings + assert '"custom_external_tool"' in settings + assert ( + runtime.files["/tmp/vf-cline/trace-id/settings/cline_mcp_settings.json"] + == b'{"mcpServers":{}}' + ) + + auth, _ = runtime.commands[-1] + assert auth[auth.index("--baseurl") + 1] == "http://interception.test/v1" + assert auth[auth.index("--modelid") + 1] == "openai/example" + + argv, env = runtime.programs[-1] + assert argv[argv.index("--key") + 1] == "ephemeral-secret" + assert argv[argv.index("--model") + 1] == "openai/example" + assert argv[argv.index("--thinking") + 1] == "medium" + assert argv[-1] == "follow the rules\n\nsolve the task" + assert env["CLINE_TELEMETRY_DISABLED"] == "1" + assert env["CLINE_MCP_SETTINGS_PATH"].endswith("cline_mcp_settings.json") + + +@pytest.mark.asyncio +async def test_cline_rejects_mcp_before_starting_program() -> None: + runtime = RecordingRuntime() + harness = ClineHarness(ClineHarnessConfig(id="cline")) + ctx = ModelContext(model="openai/example", client=EvalClientConfig()) + + with pytest.raises(ValueError, match="does not support MCP"): + await harness.launch( + ctx, + Trace.model_construct(id="trace-id"), + cast(Runtime, runtime), + "http://interception.test/v1", + "ephemeral-secret", + {"server": "http://mcp.test"}, + TaskData(prompt="solve the task"), + ) + + assert runtime.programs == [] From 940b05ad2f1cefaee17a2cb350f4f39b39662ac8 Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Sun, 23 Aug 2026 20:24:18 -0700 Subject: [PATCH 5/8] test: rely on v1 Cline e2e coverage --- tests/v1/test_cline_harness.py | 110 --------------------------------- 1 file changed, 110 deletions(-) delete mode 100644 tests/v1/test_cline_harness.py diff --git a/tests/v1/test_cline_harness.py b/tests/v1/test_cline_harness.py deleted file mode 100644 index 5295475953..0000000000 --- a/tests/v1/test_cline_harness.py +++ /dev/null @@ -1,110 +0,0 @@ -from typing import cast - -import pytest -from pydantic import ValidationError - -from verifiers.v1.clients import ModelContext -from verifiers.v1.configs.client import EvalClientConfig -from verifiers.v1.harnesses.cline import ClineHarness, ClineHarnessConfig -from verifiers.v1.runtimes import ProgramResult, Runtime -from verifiers.v1.task import TaskData -from verifiers.v1.trace import Trace -from verifiers.v1.types import Sampling - - -class RecordingRuntime: - def __init__(self) -> None: - self.commands: list[tuple[list[str], dict[str, str]]] = [] - self.programs: list[tuple[list[str], dict[str, str]]] = [] - self.files: dict[str, bytes] = {} - - async def write(self, path: str, data: bytes) -> None: - self.files[path] = data - - async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult: - self.commands.append((argv, env)) - return ProgramResult(exit_code=0, stdout="", stderr="") - - async def run_program(self, argv: list[str], env: dict[str, str]) -> ProgramResult: - self.programs.append((argv, env)) - return ProgramResult(exit_code=0, stdout="", stderr="") - - -def test_cline_config_validates_pinned_version_and_retries() -> None: - config = ClineHarnessConfig(id="cline") - assert config.version == "3.0.57" - assert config.compaction == "basic" - assert config.max_retries == 6 - - with pytest.raises(ValidationError): - ClineHarnessConfig(id="cline", version="3.0.57; unsafe") - with pytest.raises(ValidationError): - ClineHarnessConfig(id="cline", max_retries=0) - - -@pytest.mark.asyncio -async def test_cline_launch_wires_interception_and_restricts_tools() -> None: - runtime = RecordingRuntime() - harness = ClineHarness( - ClineHarnessConfig(id="cline", disabled_tools=["custom_external_tool"]) - ) - ctx = ModelContext( - model="openai/example", - client=EvalClientConfig(), - sampling=Sampling(reasoning_effort="medium"), - ) - trace = Trace.model_construct(id="trace-id") - - result = await harness.launch( - ctx, - trace, - cast(Runtime, runtime), - "http://interception.test/v1", - "ephemeral-secret", - {}, - TaskData(prompt="solve the task", system_prompt="follow the rules"), - ) - - assert result.exit_code == 0 - settings = runtime.files[ - "/tmp/vf-cline/trace-id/settings/global-settings.json" - ].decode() - assert '"fetch_web_content"' in settings - assert '"spawn_agent"' in settings - assert '"custom_external_tool"' in settings - assert ( - runtime.files["/tmp/vf-cline/trace-id/settings/cline_mcp_settings.json"] - == b'{"mcpServers":{}}' - ) - - auth, _ = runtime.commands[-1] - assert auth[auth.index("--baseurl") + 1] == "http://interception.test/v1" - assert auth[auth.index("--modelid") + 1] == "openai/example" - - argv, env = runtime.programs[-1] - assert argv[argv.index("--key") + 1] == "ephemeral-secret" - assert argv[argv.index("--model") + 1] == "openai/example" - assert argv[argv.index("--thinking") + 1] == "medium" - assert argv[-1] == "follow the rules\n\nsolve the task" - assert env["CLINE_TELEMETRY_DISABLED"] == "1" - assert env["CLINE_MCP_SETTINGS_PATH"].endswith("cline_mcp_settings.json") - - -@pytest.mark.asyncio -async def test_cline_rejects_mcp_before_starting_program() -> None: - runtime = RecordingRuntime() - harness = ClineHarness(ClineHarnessConfig(id="cline")) - ctx = ModelContext(model="openai/example", client=EvalClientConfig()) - - with pytest.raises(ValueError, match="does not support MCP"): - await harness.launch( - ctx, - Trace.model_construct(id="trace-id"), - cast(Runtime, runtime), - "http://interception.test/v1", - "ephemeral-secret", - {"server": "http://mcp.test"}, - TaskData(prompt="solve the task"), - ) - - assert runtime.programs == [] From 42684222e95d3d97bf3cb85e294404af9b0ffd28 Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Sun, 23 Aug 2026 21:47:41 -0700 Subject: [PATCH 6/8] fix(cline): preserve native tool defaults --- verifiers/v1/harnesses/cline/README.md | 5 ++-- verifiers/v1/harnesses/cline/harness.py | 33 +------------------------ 2 files changed, 3 insertions(+), 35 deletions(-) diff --git a/verifiers/v1/harnesses/cline/README.md b/verifiers/v1/harnesses/cline/README.md index 1f628c6c3c..51d3c3e159 100644 --- a/verifiers/v1/harnesses/cline/README.md +++ b/verifiers/v1/harnesses/cline/README.md @@ -32,7 +32,7 @@ id = "cline" version = "3.0.57" compaction = "basic" # basic | agentic | off max_retries = 6 -disabled_tools = [] # additional native Cline tool names +disabled_tools = [] # native Cline tool names to disable ``` Use the agent and sampling blocks for settings owned by verifiers: @@ -59,8 +59,7 @@ Cline's requests before forwarding them upstream. - Cline telemetry and update checks are disabled. - Cline MCP configuration starts empty. The harness does not advertise MCP or resume support yet. -- Network/human/nested-agent tools are disabled by default. The retained tools - read, search, edit, and run commands in the task runtime. +- Native Cline tools remain enabled unless they are listed in `disabled_tools`. - The adapter uses Cline's one-shot headless mode. Multi-turn tool use inside one task is captured, but env-driven user simulation requires future ACP support. - Cline `3.0.57` has no max-step flag. Use verifiers' `max_turns`, token caps, and diff --git a/verifiers/v1/harnesses/cline/harness.py b/verifiers/v1/harnesses/cline/harness.py index 6ed08c4e98..1a42c2da69 100644 --- a/verifiers/v1/harnesses/cline/harness.py +++ b/verifiers/v1/harnesses/cline/harness.py @@ -25,34 +25,6 @@ CLINE_BIN = f"{PACKAGES_DIR}/node_modules/cline/bin/cline" CLINE_DATA_ROOT = "/tmp/vf-cline" -# Keep the initial coding harness local to the task runtime. These tools add -# external information, human interaction, or nested agents whose work would not -# be represented as the primary harness's ordinary tool loop. -RESTRICTED_TOOLS = ( - "fetch_web_content", - "skills", - "ask_question", - "spawn_agent", - "team_spawn_teammate", - "team_shutdown_teammate", - "team_status", - "team_task", - "team_run_task", - "team_cancel_run", - "team_list_runs", - "team_await_runs", - "team_send_message", - "team_broadcast", - "team_read_mailbox", - "team_mission_log", - "team_cleanup", - "team_create_outcome", - "team_attach_outcome_fragment", - "team_review_outcome_fragment", - "team_finalize_outcome", - "team_list_outcomes", -) - INSTALL = r""" set -e export PATH="/var/tmp/vf-node/bin:$PATH" @@ -119,12 +91,9 @@ async def launch( data_dir = self.data_dir(trace) settings_dir = f"{data_dir}/settings" mcp_settings = f"{settings_dir}/cline_mcp_settings.json" - disabled_tools = list( - dict.fromkeys([*RESTRICTED_TOOLS, *(self.config.disabled_tools or [])]) - ) await runtime.write( f"{settings_dir}/global-settings.json", - json.dumps({"disabledTools": disabled_tools}).encode(), + json.dumps({"disabledTools": self.config.disabled_tools or []}).encode(), ) await runtime.write(mcp_settings, b'{"mcpServers":{}}') From 665088498e1d14a699b203ecba8448672f2671c1 Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Wed, 26 Aug 2026 13:44:58 -0700 Subject: [PATCH 7/8] feat(cline): use native ACP harness --- tests/v1/test_cline_harness.py | 49 ++++++++++++++ tests/v1/test_e2e.py | 1 + verifiers/v1/harnesses/cline/README.md | 19 +++--- verifiers/v1/harnesses/cline/harness.py | 87 +++++++++++++------------ 4 files changed, 105 insertions(+), 51 deletions(-) create mode 100644 tests/v1/test_cline_harness.py diff --git a/tests/v1/test_cline_harness.py b/tests/v1/test_cline_harness.py new file mode 100644 index 0000000000..ea01872ba9 --- /dev/null +++ b/tests/v1/test_cline_harness.py @@ -0,0 +1,49 @@ +import json +from types import SimpleNamespace +from typing import cast +from unittest.mock import AsyncMock + +import pytest + +from verifiers.v1.clients import ModelContext +from verifiers.v1.harnesses.cline.harness import ClineHarness, ClineHarnessConfig +from verifiers.v1.runtimes import ProgramResult +from verifiers.v1.task import TaskData +from verifiers.v1.trace import Trace + + +@pytest.mark.asyncio +async def test_prepare_acp_allows_user_to_open_conversation(): + harness = ClineHarness(ClineHarnessConfig(id="cline")) + runtime = AsyncMock() + runtime.run.return_value = ProgramResult(exit_code=0, stdout="", stderr="") + + config = await harness.prepare_acp( + cast(ModelContext, SimpleNamespace(model="test/model")), + cast(Trace, SimpleNamespace(id="trace-id")), + runtime, + "http://model.invalid/v1", + "interception-secret", + {"resume": "http://tool.invalid/mcp"}, + TaskData(prompt=None, system_prompt="system"), + ) + + assert config.prompt is None + assert config.system_prompt == "system" + assert "--acp" in config.command + assert config.mcp_urls == {} + mcp_payload = next( + call.args[1] + for call in runtime.write.await_args_list + if call.args[0].endswith("cline_mcp_settings.json") + ) + assert json.loads(mcp_payload) == { + "mcpServers": { + "resume": { + "transport": { + "type": "streamableHttp", + "url": "http://tool.invalid/mcp", + } + } + } + } diff --git a/tests/v1/test_e2e.py b/tests/v1/test_e2e.py index c2a816c089..8c971d3732 100644 --- a/tests/v1/test_e2e.py +++ b/tests/v1/test_e2e.py @@ -66,6 +66,7 @@ def pair(a: str, b: str, id: str, *extra_marks): # retain MCP access after resuming. Cover every harness in the local container runtime, # plus remote placements for the sandbox/tunnel and native-process boundaries. ACP_RESUME_PLACEMENTS = [ + pair("cline", "docker", "cline-acp-in-docker"), pair("codex", "docker", "codex-acp-in-docker"), pair("claude-code", "docker", "claude-code-acp-in-docker"), pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"), diff --git a/verifiers/v1/harnesses/cline/README.md b/verifiers/v1/harnesses/cline/README.md index 51d3c3e159..0f0937b9a1 100644 --- a/verifiers/v1/harnesses/cline/README.md +++ b/verifiers/v1/harnesses/cline/README.md @@ -1,9 +1,9 @@ # Cline harness -The `cline` harness runs the public Cline CLI inside the selected verifiers -runtime and routes every model call through the rollout's interception server. -Cline's native OpenAI-compatible tool calls, results, usage, and final response -therefore become ordinary verifiers trace nodes and model-call records. +The `cline` harness runs the public Cline CLI through its native ACP server and +routes every model call through the rollout's interception server. Cline's tool +calls, results, usage, and final responses become ordinary verifiers trace nodes +and model-call records while one native session persists across resumed turns. ## Quickstart @@ -30,8 +30,6 @@ uv run eval MY_TASKSET \ [env.agent.harness] id = "cline" version = "3.0.57" -compaction = "basic" # basic | agentic | off -max_retries = 6 disabled_tools = [] # native Cline tool names to disable ``` @@ -57,11 +55,12 @@ Cline's requests before forwarding them upstream. - Each rollout gets an isolated Cline data directory, deleted after scoring. - Cline telemetry and update checks are disabled. -- Cline MCP configuration starts empty. The harness does not advertise MCP or - resume support yet. +- Per-rollout MCP servers are loaded through Cline's native MCP settings file. - Native Cline tools remain enabled unless they are listed in `disabled_tools`. -- The adapter uses Cline's one-shot headless mode. Multi-turn tool use inside one - task is captured, but env-driven user simulation requires future ACP support. +- The adapter keeps one native ACP session alive across env-driven user turns. +- Cline exposes synchronous `tool_call` and `tool_result` hooks, but this harness + does not advertise verifiers tool interception because the public hook control + surface does not preserve the full result-rewrite contract. - Cline `3.0.57` has no max-step flag. Use verifiers' `max_turns`, token caps, and rollout timeout; their stop conditions and errors are recorded on the trace. - Eval-client traces preserve message/tool structure and provider-reported usage, diff --git a/verifiers/v1/harnesses/cline/harness.py b/verifiers/v1/harnesses/cline/harness.py index 1a42c2da69..5668d8c80b 100644 --- a/verifiers/v1/harnesses/cline/harness.py +++ b/verifiers/v1/harnesses/cline/harness.py @@ -1,17 +1,16 @@ -"""Run the public Cline CLI headlessly through interception.""" +"""Run the public Cline CLI through its native ACP server.""" import json import logging import shlex -from typing import Literal from pydantic import Field +from verifiers.v1.acp import ACPConfig, ACPHarness from verifiers.v1.clients import ModelContext from verifiers.v1.configs.harness import HarnessConfig -from verifiers.v1.harness import Harness from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node -from verifiers.v1.runtimes import ProgramResult, Runtime +from verifiers.v1.runtimes import Runtime from verifiers.v1.task import TaskData from verifiers.v1.trace import Trace @@ -39,14 +38,11 @@ class ClineHarnessConfig(HarnessConfig): version: str = Field(default="3.0.57", pattern=r"^[A-Za-z0-9._+-]+$") """Public Cline CLI release to install, pinned for reproducibility.""" - compaction: Literal["agentic", "basic", "off"] = "basic" - """Cline's context-compaction mode.""" - max_retries: int = Field(default=6, ge=1) - """Maximum consecutive Cline mistakes before the CLI exits.""" +class ClineHarness(ACPHarness[ClineHarnessConfig]): + APPENDS_SYSTEM_PROMPT = True + SUPPORTS_MCP = True - -class ClineHarness(Harness[ClineHarnessConfig]): async def setup(self, runtime: Runtime) -> None: await ensure_node(runtime) directory = CLINE_DIR.format(version=self.config.version) @@ -71,8 +67,9 @@ async def setup(self, runtime: Runtime) -> None: if result.exit_code != 0: detail = (result.stderr or result.stdout).strip()[-500:] raise RuntimeError(f"Cline CLI install failed: {detail}") + await super().setup(runtime) - async def launch( + async def prepare_acp( self, ctx: ModelContext, trace: Trace, @@ -81,12 +78,8 @@ async def launch( secret: str, mcp_urls: dict[str, str], data: TaskData, - ) -> ProgramResult: - if mcp_urls: - raise ValueError("Cline harness v1 does not support MCP servers") - _, prompt = self.resolve_text_prompt(data) - if prompt is None: - raise ValueError("Cline requires a task prompt") + ) -> ACPConfig: + system_prompt, prompt = self.resolve_prompt(data) data_dir = self.data_dir(trace) settings_dir = f"{data_dir}/settings" @@ -95,7 +88,22 @@ async def launch( f"{settings_dir}/global-settings.json", json.dumps({"disabledTools": self.config.disabled_tools or []}).encode(), ) - await runtime.write(mcp_settings, b'{"mcpServers":{}}') + await runtime.write( + mcp_settings, + json.dumps( + { + "mcpServers": { + name: { + "transport": { + "type": "streamableHttp", + "url": url, + } + } + for name, url in mcp_urls.items() + } + } + ).encode(), + ) env = { **self.config.resolved_env, @@ -105,6 +113,9 @@ async def launch( ), "CLINE_DATA_DIR": data_dir, "CLINE_MCP_SETTINGS_PATH": mcp_settings, + "CLINE_PROVIDER": "openai-compatible", + "CLINE_API_KEY": secret, + "CLINE_MODEL": ctx.model, "CLINE_TELEMETRY_DISABLED": "1", "CLINE_NO_AUTO_UPDATE": "1", "NO_UPDATE_NOTIFIER": "1", @@ -134,29 +145,23 @@ async def launch( detail = (auth.stderr or auth.stdout).strip()[-500:] raise RuntimeError(f"Cline provider configuration failed: {detail}") - args = [ - *cline, - "--json", - "--auto-approve", - "true", - "--cwd", - ".", - "--provider", - "openai-compatible", - "--key", - secret, - "--model", - ctx.model, - "--compaction", - self.config.compaction, - "--retries", - str(self.config.max_retries), - "--data-dir", - data_dir, - ] - if effort := ctx.sampling.reasoning_effort: - args += ["--thinking", effort] - return await runtime.run_program([*args, prompt], env) + return ACPConfig( + env=env, + command=[ + *cline, + "--acp", + "--auto-approve", + "true", + "--cwd", + ".", + "--data-dir", + data_dir, + ], + prompt=prompt, + # Cline reads task-scoped servers from CLINE_MCP_SETTINGS_PATH. + mcp_urls={}, + system_prompt=system_prompt, + ) async def cleanup(self, trace: Trace, runtime: Runtime) -> None: result = await runtime.run(["rm", "-rf", self.data_dir(trace)], {}) From d539ef0cbc871e39ca75bc96cc53a3718a333895 Mon Sep 17 00:00:00 2001 From: Arafatkatze Date: Sat, 29 Aug 2026 15:22:58 -0700 Subject: [PATCH 8/8] fix(cline): address harness review feedback --- tests/v1/test_cline_harness.py | 49 ------------------------- verifiers/v1/harnesses/cline/harness.py | 11 ++++-- 2 files changed, 8 insertions(+), 52 deletions(-) delete mode 100644 tests/v1/test_cline_harness.py diff --git a/tests/v1/test_cline_harness.py b/tests/v1/test_cline_harness.py deleted file mode 100644 index ea01872ba9..0000000000 --- a/tests/v1/test_cline_harness.py +++ /dev/null @@ -1,49 +0,0 @@ -import json -from types import SimpleNamespace -from typing import cast -from unittest.mock import AsyncMock - -import pytest - -from verifiers.v1.clients import ModelContext -from verifiers.v1.harnesses.cline.harness import ClineHarness, ClineHarnessConfig -from verifiers.v1.runtimes import ProgramResult -from verifiers.v1.task import TaskData -from verifiers.v1.trace import Trace - - -@pytest.mark.asyncio -async def test_prepare_acp_allows_user_to_open_conversation(): - harness = ClineHarness(ClineHarnessConfig(id="cline")) - runtime = AsyncMock() - runtime.run.return_value = ProgramResult(exit_code=0, stdout="", stderr="") - - config = await harness.prepare_acp( - cast(ModelContext, SimpleNamespace(model="test/model")), - cast(Trace, SimpleNamespace(id="trace-id")), - runtime, - "http://model.invalid/v1", - "interception-secret", - {"resume": "http://tool.invalid/mcp"}, - TaskData(prompt=None, system_prompt="system"), - ) - - assert config.prompt is None - assert config.system_prompt == "system" - assert "--acp" in config.command - assert config.mcp_urls == {} - mcp_payload = next( - call.args[1] - for call in runtime.write.await_args_list - if call.args[0].endswith("cline_mcp_settings.json") - ) - assert json.loads(mcp_payload) == { - "mcpServers": { - "resume": { - "transport": { - "type": "streamableHttp", - "url": "http://tool.invalid/mcp", - } - } - } - } diff --git a/verifiers/v1/harnesses/cline/harness.py b/verifiers/v1/harnesses/cline/harness.py index 5668d8c80b..2c17bf0f72 100644 --- a/verifiers/v1/harnesses/cline/harness.py +++ b/verifiers/v1/harnesses/cline/harness.py @@ -86,7 +86,12 @@ async def prepare_acp( mcp_settings = f"{settings_dir}/cline_mcp_settings.json" await runtime.write( f"{settings_dir}/global-settings.json", - json.dumps({"disabledTools": self.config.disabled_tools or []}).encode(), + json.dumps( + { + "disabledTools": self.config.disabled_tools or [], + "telemetryOptOut": True, + } + ).encode(), ) await runtime.write( mcp_settings, @@ -97,7 +102,8 @@ async def prepare_acp( "transport": { "type": "streamableHttp", "url": url, - } + }, + "timeout": self.config.tool_timeout, } for name, url in mcp_urls.items() } @@ -116,7 +122,6 @@ async def prepare_acp( "CLINE_PROVIDER": "openai-compatible", "CLINE_API_KEY": secret, "CLINE_MODEL": ctx.model, - "CLINE_TELEMETRY_DISABLED": "1", "CLINE_NO_AUTO_UPDATE": "1", "NO_UPDATE_NOTIFIER": "1", }