Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 18 additions & 21 deletions src/inspect_swe/_claude_code/claude_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ def claude_code(
session_id = str(uuid.uuid4())

# each --resume rebuilds the system prompt from the current environment
# (regenerated git status, dropped --append-system-prompt text), busting
# (regenerated git status), busting
# the prompt cache for the whole conversation prefix and leaking that the
# session was restarted -- pin the root conversation's system prompt to
# its first-seen value (see systemprompt.py). Built once per agent
Expand Down Expand Up @@ -424,6 +424,22 @@ async def execute(state: AgentState) -> AgentState:
cmd.append("--disallowed-tools")
cmd.append(",".join(disallowed_tools))

# Claude Code keeps neither system prompt flag in its session, so
# every launch, --resume included, passes whichever is set. The caller's system
# messages are recorded before the session's first launch: the
# bridge then replaces them in state.messages with Claude Code's
# own prompt, which must not be appended to itself.
appended_key = f"claude_code_appended_system_prompt:{session_id}"
appended_system: list[str] | None = store().get(appended_key, None)
if appended_system is None:
appended_system = [
m.text for m in state.messages if isinstance(m, ChatMessageSystem)
]
if system_prompt is not None:
appended_system.append(system_prompt)
store().set(appended_key, appended_system)
system_args = _system_prompt_args(appended_system, replace_system_prompt)

prompt, has_assistant_response = build_user_prompt(state.messages)

# resolve sandbox
Expand Down Expand Up @@ -489,23 +505,6 @@ async def execute(state: AgentState) -> AgentState:
or cp.attempt == "resume"
)

# Replacement flags are per-invocation, so re-send them on
# resume. Appended messages are not re-sent because the bridge
# round-trips them into state.messages and appending them again
# would duplicate the effective prompt.
system_texts = [
m.text
for m in state.messages
if isinstance(m, ChatMessageSystem)
]
if system_prompt is not None:
system_texts.append(system_prompt)
system_args = _system_prompt_args(
system_texts,
replace_system_prompt,
is_resume=is_resume,
)

# resume previous conversation
if is_resume:
agent_cmd = (
Expand Down Expand Up @@ -667,13 +666,11 @@ async def execute(state: AgentState) -> AgentState:
def _system_prompt_args(
system_texts: Sequence[str],
replace_system_prompt: str | None,
*,
is_resume: bool,
) -> list[str]:
args: list[str] = []
if replace_system_prompt is not None:
args.extend(["--system-prompt", replace_system_prompt])
if system_texts and not is_resume:
if system_texts:
args.extend(["--append-system-prompt", "\n\n".join(system_texts)])

return args
Expand Down
5 changes: 3 additions & 2 deletions src/inspect_swe/_claude_code/systemprompt.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,9 @@
regenerated from the CURRENT repo state, so the agent sees its own
mid-episode edits presented as the starting state -- agents have used
the contradiction to recognize they are running inside an evaluation;
- ``--append-system-prompt`` text is only passed on the first launch, so
it disappears on resume and the identity line flips.
- ``--append-system-prompt`` text is not kept in the session; without it a
resume drops that text and flips the identity line, so claude_code.py
passes it on every launch (verified against Claude Code 2.1.236).

The system prompt heads every request and carries a cache_control
breakpoint, so any changed byte invalidates the prompt cache for the whole
Expand Down
126 changes: 109 additions & 17 deletions tests/test_claude_code_system_prompt.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,38 @@
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, Mock

import pytest
from inspect_ai import Task, eval
from inspect_ai.agent import AgentState, as_solver
from inspect_ai.dataset import Sample
from inspect_ai.model import ChatMessageSystem, ChatMessageUser
from inspect_ai.scorer import INCORRECT, Score
from inspect_ai.util import ExecCompleted, ExecRemoteProcess, SandboxEnvironment
from inspect_swe import claude_code
from inspect_swe._claude_code import claude_code as claude_code_module
from inspect_swe._claude_code.claude_code import _system_prompt_args


def test_system_prompt_appends_to_default() -> None:
assert _system_prompt_args(
["Task prompt", "Agent prompt"], None, is_resume=False
) == [
assert _system_prompt_args(["Task prompt", "Agent prompt"], None) == [
"--append-system-prompt",
"Task prompt\n\nAgent prompt",
]


def test_system_prompt_can_replace_default() -> None:
assert _system_prompt_args([], "Replacement prompt", is_resume=False) == [
assert _system_prompt_args([], "Replacement prompt") == [
"--system-prompt",
"Replacement prompt",
]


def test_task_prompt_is_appended_to_replacement() -> None:
assert _system_prompt_args(
["Task prompt"], "Replacement prompt", is_resume=False
) == [
assert _system_prompt_args(["Task prompt"], "Replacement prompt") == [
"--system-prompt",
"Replacement prompt",
"--append-system-prompt",
Expand All @@ -31,16 +41,7 @@ def test_task_prompt_is_appended_to_replacement() -> None:


def test_empty_system_prompts_add_no_cli_flags() -> None:
assert _system_prompt_args([], None, is_resume=False) == []


def test_resume_reapplies_replacement_without_appended_messages() -> None:
assert _system_prompt_args(
["Round-tripped prompt"], "Replacement prompt", is_resume=True
) == [
"--system-prompt",
"Replacement prompt",
]
assert _system_prompt_args([], None) == []


def test_append_and_replace_system_prompts_are_mutually_exclusive() -> None:
Expand All @@ -52,3 +53,94 @@ def test_append_and_replace_system_prompts_are_mutually_exclusive() -> None:
system_prompt="Additional prompt",
replace_system_prompt="Replacement prompt",
)


class _ExitedProcess(ExecRemoteProcess):
"""A Claude Code launch that exits 0 without output."""

def __init__(self) -> None:
self._exited = False

def __aiter__(self) -> "_ExitedProcess":
return self

async def __anext__(self) -> ExecCompleted:
if self._exited:
raise StopAsyncIteration
self._exited = True
return ExecCompleted(0)


def test_every_launch_appends_the_callers_system_prompt(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
"""A resumed launch re-sends the caller's system prompt, once.

Claude Code does not keep --append-system-prompt in its session, so a
--resume without it runs on Claude Code's default prompt alone. The bridge
replaces the state's system messages with Claude Code's own prompt after a
launch; re-sending those would duplicate it.
"""
commands: list[list[str]] = []

@asynccontextmanager
async def bridge(state: AgentState, **kwargs: Any) -> AsyncIterator[Any]:
async def exec_remote(cmd: list[str], **kwargs: Any) -> ExecRemoteProcess:
commands.append(cmd)
state.messages = [
ChatMessageSystem(content="Claude Code's own prompt\n\nTask prompt"),
*[m for m in state.messages if not isinstance(m, ChatMessageSystem)],
]
return _ExitedProcess()

sandbox.exec_remote = exec_remote
yield SimpleNamespace(state=state, port=3001, mcp_server_configs=[])

@asynccontextmanager
async def checkpoint() -> AsyncIterator[Any]:
yield SimpleNamespace(
attempt="start", track=lambda name, getter, initial: initial
)

sandbox = Mock(spec=SandboxEnvironment)
module = claude_code_module
monkeypatch.setattr(module, "sandbox_agent_bridge", bridge)
monkeypatch.setattr(module, "checkpointer", checkpoint)
monkeypatch.setattr(module, "sandbox_env", lambda _: sandbox)
monkeypatch.setattr(
module, "ensure_agent_binary_installed", AsyncMock(return_value="claude")
)
monkeypatch.setattr(
module, "resolve_agent_cwd", AsyncMock(return_value=str(tmp_path))
)
monkeypatch.setattr(module, "_seed_claude_config", AsyncMock())
monkeypatch.setattr(
module, "score", AsyncMock(return_value=[Score(value=INCORRECT)])
)

agent = claude_code(attempts=2)
log = eval(
Task(
dataset=[
Sample(
input=[
ChatMessageSystem(content="Task prompt"),
ChatMessageUser(content="Solve it."),
]
)
],
# two turns of one agent: each turn's attempts relaunch with
# --resume, and the second turn starts from the bridged state
solver=[as_solver(agent), as_solver(agent)],
),
model="mockllm/model",
display="none",
log_dir=str(tmp_path),
)[0]

assert log.status == "success", log.error
assert len(commands) == 4
assert "--resume" in commands[1] and "--resume" in commands[3]
for command in commands:
flags = [i for i, arg in enumerate(command) if arg == "--append-system-prompt"]
assert [command[i + 1] for i in flags] == ["Task prompt"]