From 12942d091434f6fa947a4ff8ef10e76a96819d76 Mon Sep 17 00:00:00 2001 From: YaoCheng Date: Fri, 4 Sep 2026 05:09:36 +0000 Subject: [PATCH] feat(docker): control-plane container image and --docker CLI mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a lightweight Docker image that carries agent_sys, Claude Code CLI, and standard tools (git, ssh, rsync, uv) but no inference engines or task-specific dependencies. Those are installed by env_mgr at runtime per each task package's environment recipe. Key changes: Docker image (deploy/docker/) - Multi-stage Dockerfile: base → toolchain → venv-agent → runtime. Magpie, compilers, and model weights are excluded by design. - build.sh: thin wrapper around docker build with --tag support. - README.md: build, usage (manual and --docker), and exit codes. Container lifecycle (agent_sys/env_mgr/container.py) - ContainerManager: build, start, exec, stop. - Auto-detects and bind-mounts host ~/.ssh (ro) and ~/.claude. - Generates /etc/passwd and /etc/group for arbitrary UID/GID. CLI --docker mode (agent_sys/cli/main.py) - --docker, --docker-image, --detect-and-copy-host-{ssh,claude}-config. - Delegates the run into the container, stripping docker-specific flags. Work root (agent_sys/cli/environment.py) - INFERA_AGENT_SYSTEM_WORKROOT env var: names the run root directly. - Falls back to $XDG_STATE_HOME/agent-sys-demo when unset. - Empty or relative values are treated as unset. Executor shutdown (agent_sys/agent/) - claude_sdk.py: deterministic teardown -- disconnect, cancel pending tasks, close the private event loop. Idempotent. - runner.py: Runner.shutdown() releases all surviving attempts. - cli/main.py: outer try/finally calls runner.shutdown() after the run report, preventing leaked SDK reader tasks at process exit. Tests - test_claude_sdk.py: stop disconnects, cancels, and closes the loop. - test_runner.py: shutdown stops and forgets every surviving attempt. - test_shutdown.py: CLI-level shutdown is called on both success and error paths. - test_build.py: INFERA_AGENT_SYSTEM_WORKROOT precedence, relative and empty values fall back correctly. Co-authored-by: Cursor --- agent_sys/agent/backends/claude_sdk.py | 40 ++- agent_sys/agent/runner.py | 39 ++- agent_sys/cli/environment.py | 38 ++- agent_sys/cli/main.py | 175 +++++++++++-- agent_sys/env_mgr/container.py | 236 ++++++++++++++++++ agent_sys/tests/agent/conftest.py | 4 + agent_sys/tests/agent/test_claude_sdk.py | 40 +++ agent_sys/tests/agent/test_runner.py | 24 ++ agent_sys/tests/cli/test_build.py | 54 ++++ agent_sys/tests/cli/test_shutdown.py | 69 +++++ deploy/docker/Dockerfile.agent-sys | 140 +++++++++++ .../docker/Dockerfile.agent-sys.dockerignore | 34 +++ deploy/docker/agent-sys/README.md | 105 ++++++++ deploy/docker/agent-sys/build.sh | 35 +++ 14 files changed, 1006 insertions(+), 27 deletions(-) create mode 100644 agent_sys/env_mgr/container.py create mode 100644 agent_sys/tests/cli/test_shutdown.py create mode 100644 deploy/docker/Dockerfile.agent-sys create mode 100644 deploy/docker/Dockerfile.agent-sys.dockerignore create mode 100644 deploy/docker/agent-sys/README.md create mode 100755 deploy/docker/agent-sys/build.sh diff --git a/agent_sys/agent/backends/claude_sdk.py b/agent_sys/agent/backends/claude_sdk.py index 3c4e8663e..a32b94827 100644 --- a/agent_sys/agent/backends/claude_sdk.py +++ b/agent_sys/agent/backends/claude_sdk.py @@ -466,9 +466,45 @@ def _deliver(self, message: str) -> None: self._await(self._client.query(message)) def _terminate(self) -> None: - if self._connected: - self._await(self._client.disconnect()) + """Disconnect the SDK and deterministically dispose of its private loop. + + A successful submission deliberately leaves the client connected: the + completeness gate may request another submission on the same session. + `stop()` is the owner boundary where that reuse has ended. Merely + dropping the loop here leaves the SDK's `Query._read_messages` task and + subprocess transport alive; Python then destroys them after the loop is + closed and emits `Task was destroyed but it is pending`. + + Idempotence matters because scheduler stop and process shutdown can both + reach the same executor. + """ + with self._loop_lock: + if self._loop.is_closed(): + self._connected = False + return + try: + self._loop.run_until_complete(self._shutdown()) + finally: + self._loop.close() + + async def _shutdown(self) -> None: + """Close the SDK first, then cancel anything it left on this loop.""" + try: + if self._connected: + await self._client.disconnect() + finally: self._connected = False + current = asyncio.current_task() + pending = [ + task + for task in asyncio.all_tasks(self._loop) + if task is not current and not task.done() + ] + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + await self._loop.shutdown_asyncgens() # ---- level 2 ---------------------------------------------------------- # diff --git a/agent_sys/agent/runner.py b/agent_sys/agent/runner.py index 2acd8d58a..e9ec75fa4 100644 --- a/agent_sys/agent/runner.py +++ b/agent_sys/agent/runner.py @@ -182,6 +182,36 @@ def stop(self, task_id: TaskId, on_stopped: Callable[[TaskId], None]) -> None: self._attempts.pop(task_id, None) on_stopped(task_id) + def shutdown(self) -> None: + """Release every attempt when the process-level run is over. + + Attempts outlive their worker thread on purpose: a monitor can wake a + parked task or submit to the same executor again while the run exists. + Once all monitors have stopped and the CLI has produced its report, + there is no remaining owner for those executors. Keeping them in + `_attempts` leaked each Claude SDK reader task and subprocess transport + until interpreter teardown. + + The map is emptied before any potentially blocking backend shutdown, so + no concurrent observer can acquire an executor whose disposal started. + Every attempt is given a chance to stop even when one backend raises; + the first error is re-raised after all worker threads have been joined. + """ + with self._lock: + attempts = tuple(self._attempts.values()) + self._attempts.clear() + + first_error: Exception | None = None + for attempt in attempts: + try: + attempt.halt() + except Exception as exc: # cleanup all owners before reporting one failure + first_error = first_error or exc + for attempt in attempts: + attempt.join(HANDOVER_GRACE) + if first_error is not None: + raise first_error + # ---- the monitor's two entrances, not on the protocol ----------------- # # `Runner.resume` is gone. `carry_on` subsumed it, `monitor`'s Protocol @@ -560,9 +590,12 @@ def halt(self) -> None: """`Runner.stop`: end the executor and the thread.""" with self._own: self._halted = True - if self.executor is not None: - self.executor.stop() - self._wake.set() + try: + if self.executor is not None: + self.executor.stop() + finally: + # A backend cleanup error must not leave the attempt parked forever. + self._wake.set() def join(self, timeout: float | None = None) -> None: """For a caller that wants the thread settled. Tests, and `demo`.""" diff --git a/agent_sys/cli/environment.py b/agent_sys/cli/environment.py index c70fff411..f2ad77f76 100644 --- a/agent_sys/cli/environment.py +++ b/agent_sys/cli/environment.py @@ -45,7 +45,9 @@ "Layout", "LiveHandoffs", "build_context", + "WORKROOT_ENV_VAR", "confinement", + "default_root", "demo_grants", "latest_run", "layout_for", @@ -214,14 +216,48 @@ def __len__(self) -> int: return 0 if self._mgr is None else len(self._mgr.all_ids()) +#: The run root, named outright. `--demo-root` still wins; this is what a +#: caller who cannot pass an argument sets. +#: +#: **It names the root itself, not a base to append to.** `XDG_STATE_HOME` is a +#: base — the spec says what may go under it and every program appends its own +#: name — so `agent-sys-demo` is this program's share of a directory it does not +#: own. That is the wrong shape for the thing this variable exists to control: +#: the run root has to be **one absolute path that resolves identically here and +#: on the compute node**, because `remote.sh:require_visible_on_node` asserts +#: exactly that and the bodies run out of `/runs//zones/…/package` +#: by absolute path. A variable that only moves the parent leaves the operator +#: composing the real answer in their head, and a container bind mount has to +#: name the whole path anyway. +#: +#: `INFERA_` and not `AGENT_SYS_`: the `AGENT_SYS_*` namespace is what `env_mgr` +#: *publishes to a body* — `AGENT_SYS_MY_ZONE`, `AGENT_SYS_MY_WORKSPACE`, +#: `AGENT_SYS_TASK_PACKAGE` — and one of those set by a caller is either ignored +#: or a collision. A variable read *from* the environment does not belong in a +#: namespace whose other members are written *to* it. +WORKROOT_ENV_VAR = "INFERA_AGENT_SYSTEM_WORKROOT" + + def default_root() -> Path: - """`$XDG_STATE_HOME/agent-sys-demo`, or `~/.local/state/...`. + """`$INFERA_AGENT_SYSTEM_WORKROOT`, else `$XDG_STATE_HOME/agent-sys-demo`. + + Two sources and one rule: the specific name wins over the generic base. A + caller who set neither gets `~/.local/state/agent-sys-demo`. State rather than cache or data: it is *"state that should persist between restarts but is not important enough for the data directory"*, which is what a demo run is. A cache directory would be correct until somebody cleared it between the interrupt and the resume. + + **An empty or relative value reads as unset.** `XDG_STATE_HOME`'s own + specification says exactly that of a base directory, and the reason applies + with more force here: a relative run root resolves against whatever `cwd` a + body inherited, which is the one thing `` may not depend on if the + compute node is to find the same directory. """ + named = os.environ.get(WORKROOT_ENV_VAR, "").strip() + if named and os.path.isabs(named): + return Path(named) base = os.environ.get("XDG_STATE_HOME") or str(Path.home() / ".local" / "state") return Path(base) / "agent-sys-demo" diff --git a/agent_sys/cli/main.py b/agent_sys/cli/main.py index 0acd84a24..ef22a0ed7 100644 --- a/agent_sys/cli/main.py +++ b/agent_sys/cli/main.py @@ -173,6 +173,47 @@ def parser() -> argparse.ArgumentParser: "ends in seconds regardless; this only bounds one that never stops" ), ) + + # Docker mode: run inside a container managed by env_mgr. + run.add_argument( + "--docker", + action="store_true", + help="run inside a Docker container (env_mgr builds and starts it automatically)", + ) + run.add_argument( + "--docker-debug", + action="store_true", + help="start the container and drop into an interactive shell (sleep infinity), no task is run", + ) + run.add_argument( + "--docker-image", + metavar="IMAGE", + default="infera/agent-sys:latest", + help="Docker image to use (default: infera/agent-sys:latest)", + ) + run.add_argument( + "--docker-name", + metavar="NAME", + default="agent-sys-container", + help="container name (default: agent-sys-container)", + ) + run.add_argument( + "--docker-rm", + action="store_true", + help="remove the container after the run finishes (default: keep running)", + ) + run.add_argument( + "--detect-and-copy-host-ssh-config", + action=argparse.BooleanOptionalAction, + default=True, + help="mount host ~/.ssh into the container (default: on)", + ) + run.add_argument( + "--detect-and-copy-host-claude-config", + action=argparse.BooleanOptionalAction, + default=True, + help="mount host ~/.claude into the container (default: on)", + ) return top @@ -262,11 +303,97 @@ def _show(args: argparse.Namespace, stream: Stream) -> int: def _run(args: argparse.Namespace, stream: Stream) -> int: if args.clean: return _clean(args, stream) + if getattr(args, "docker_debug", False): + return _docker_debug(args, stream) + if getattr(args, "docker", False): + return _docker_run(args, stream) if args.dry_run: return _dry_run(args, stream) return _real_run(args, stream) +def _docker_debug(args: argparse.Namespace, stream: Stream) -> int: + """Start a container with sleep infinity and drop into a shell.""" + from env_mgr.container import ContainerManager + + mgr = ContainerManager( + image=args.docker_image, + container_name=args.docker_name, + detect_ssh=args.detect_and_copy_host_ssh_config, + detect_claude=args.detect_and_copy_host_claude_config, + ) + + mgr.start() + + name = mgr.container_name + stream.emit( + EventKind.RUN_COMPLETE, + f"container '{name}' is running (sleep infinity). Connect with:\n" + f"\n" + f" docker exec -it {name} bash\n", + exit_code=OK, + ok=True, + ) + + import subprocess + return subprocess.run( + [mgr._docker_bin(), "exec", "-it", "-w", "/opt/Infera", name, "bash"], + ).returncode + + +def _docker_run(args: argparse.Namespace, stream: Stream) -> int: + """Delegate the run to a Docker container managed by env_mgr.""" + from env_mgr.container import ContainerManager + + mgr = ContainerManager( + image=args.docker_image, + container_name=args.docker_name, + detect_ssh=args.detect_and_copy_host_ssh_config, + detect_claude=args.detect_and_copy_host_claude_config, + ) + + stream.emit( + EventKind.RUN_COMPLETE, + f"starting Docker container (image={mgr.image})", + exit_code=OK, + ok=True, + ) + + mgr.start() + + # Forward the original command into the container, stripping --docker flags. + forwarded = ["agent-sys", "run"] + if args.package: + forwarded += ["--package", args.package] + if args.demo_root: + forwarded += ["--demo-root", args.demo_root] + if args.dry_run: + forwarded += ["--dry-run"] + if getattr(args, "with_broken", False): + forwarded += ["--with-broken"] + if getattr(args, "resume", False): + forwarded += ["--resume"] + if getattr(args, "allow_repo_config", False): + forwarded += ["--allow-repo-config"] + for var_item in args.var: + forwarded += ["--var", var_item] + if args.json: + forwarded += ["--json", args.json] + + rc = mgr.exec(forwarded, workdir="/opt/Infera") + + if getattr(args, "docker_rm", False): + mgr.stop() + stream.emit( + EventKind.RUN_COMPLETE, + f"container '{mgr.container_name}' removed", + exit_code=rc, + ok=rc == 0, + ) + + return rc + + def _clean(args: argparse.Namespace, stream: Stream) -> int: layout = layout_for(Path(args.demo_root) if args.demo_root else None) shutil.rmtree(layout.root / "runs", ignore_errors=True) @@ -398,28 +525,34 @@ def _real_run(args: argparse.Namespace, stream: Stream) -> int: # which is how the bug was found. One call now. running = start_monitors(registry) try: - if args.resume: - resume_all(registry) - else: - _start(registry, stream) - _settle(registry, stream, timeout=getattr(args, "timeout", None) or _SETTLE_TIMEOUT) + try: + if args.resume: + resume_all(registry) + else: + _start(registry, stream) + _settle(registry, stream, timeout=getattr(args, "timeout", None) or _SETTLE_TIMEOUT) + finally: + # Names that did **not** come back, rather than a hang or a silent pass. + stragglers = running.stop(timeout=5.0) + if stragglers: + stream.emit( + EventKind.RUN_COMPLETE, + f"monitor loops that did not return: {sorted(stragglers)}", + stragglers=sorted(stragglers), + ok=False, + ) + # Described AFTER the run, not before it: the subgraph does not exist + # until the root's main phase unfolds, so a graph printed at submit time + # would be one task long. + tasks = registry.get("task_mgr").all() + _emit_graph(stream, tasks, resumed=bool(args.resume)) + _describe(registry, tasks, stream) + return _report(registry, stream, layout, promises) finally: - # Names that did **not** come back, rather than a hang or a silent pass. - stragglers = running.stop(timeout=5.0) - if stragglers: - stream.emit( - EventKind.RUN_COMPLETE, - f"monitor loops that did not return: {sorted(stragglers)}", - stragglers=sorted(stragglers), - ok=False, - ) - # Described AFTER the run, not before it: the subgraph does not exist - # until the root's main phase unfolds, so a graph printed at submit time - # would be one task long. - tasks = registry.get("task_mgr").all() - _emit_graph(stream, tasks, resumed=bool(args.resume)) - _describe(registry, tasks, stream) - return _report(registry, stream, layout, promises) + # Monitor decisions and reporting may reuse a settled executor. Once + # both are over, every attempt belongs to this invocation and must be + # disposed before Python tears down the SDK's private event loops. + registry.get("runner").shutdown() # --------------------------------------------------------------------------- # diff --git a/agent_sys/env_mgr/container.py b/agent_sys/env_mgr/container.py new file mode 100644 index 000000000..a05f20e5d --- /dev/null +++ b/agent_sys/env_mgr/container.py @@ -0,0 +1,236 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""Container lifecycle management for agent_sys. + +Provides the ``--docker`` CLI feature: build the control-plane image when it is +absent, start a container with the right mounts, and forward the command into +it. Host SSH keys and Claude credentials are detected and bind-mounted at +runtime (never copied into the image). + +This module lives above the decoupling wall: it imports nothing from the +shipped installer subsystem and nothing from the isolation / domain subsystem. +""" + +from __future__ import annotations + +import grp +import logging +import os +import pwd +import shutil +import subprocess +import tempfile +from pathlib import Path + +__all__ = ["ContainerManager"] + +log = logging.getLogger(__name__) + +DEFAULT_IMAGE = "infera/agent-sys:latest" +DEFAULT_CONTAINER_NAME = "agent-sys-container" + + +class ContainerManager: + """Build, start, and exec into an agent_sys Docker container. + + Parameters + ---------- + image : str + Docker image tag. + container_name : str + Name for the container. + detect_ssh : bool + If True, detect and mount host ``~/.ssh`` read-only. + detect_claude : bool + If True, detect and mount host ``~/.claude``. + """ + + def __init__( + self, + *, + image: str = DEFAULT_IMAGE, + container_name: str = DEFAULT_CONTAINER_NAME, + detect_ssh: bool = True, + detect_claude: bool = True, + ) -> None: + self.image = image + self.container_name = container_name + self.detect_ssh = detect_ssh + self.detect_claude = detect_claude + + # ---------------------------------------------------------------------- # + + @staticmethod + def _docker_bin() -> str: + docker = shutil.which("docker") + if docker is None: + raise RuntimeError( + "docker is not installed or not on PATH. " + "Install Docker and try again, or run without --docker." + ) + return docker + + def image_exists(self) -> bool: + result = subprocess.run( + [self._docker_bin(), "image", "inspect", self.image], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + def container_running(self) -> bool: + result = subprocess.run( + [ + self._docker_bin(), + "container", + "inspect", + "-f", + "{{.State.Running}}", + self.container_name, + ], + capture_output=True, + text=True, + ) + return result.returncode == 0 and result.stdout.strip() == "true" + + # ---------------------------------------------------------------------- # + + def build(self, repo_root: str | Path | None = None) -> None: + """Build the control-plane image from the repository Dockerfile.""" + if repo_root is None: + repo_root = Path(__file__).resolve().parents[2] + repo_root = Path(repo_root) + dockerfile = repo_root / "deploy" / "docker" / "Dockerfile.agent-sys" + if not dockerfile.exists(): + raise FileNotFoundError(f"Dockerfile not found: {dockerfile}") + log.info("building image %s from %s", self.image, repo_root) + subprocess.run( + [ + self._docker_bin(), + "build", + "--file", + str(dockerfile), + "--tag", + self.image, + "--progress=plain", + str(repo_root), + ], + check=True, + ) + + # ---------------------------------------------------------------------- # + + def start(self, *, workroot: str | None = None) -> None: + """Start a background container, detecting host config as requested. + + If the container is already running, this is a no-op. If the image + does not exist, it is built first. + """ + if self.container_running(): + log.info("container %s is already running", self.container_name) + return + + if not self.image_exists(): + log.info("image %s not found, building it", self.image) + self.build() + + docker = self._docker_bin() + home = os.environ.get("HOME", os.path.expanduser("~")) + container_home = "/home/agent" + uid = os.getuid() + gid = os.getgid() + + if workroot is None: + workroot = os.environ.get( + "INFERA_AGENT_SYSTEM_WORKROOT", + os.path.join(home, ".agent_sys_runs"), + ) + os.makedirs(workroot, exist_ok=True) + + # A passwd/group entry so ssh's getpwuid() does not refuse. + passwd_dir = os.path.join(home, ".agent_sys") + os.makedirs(passwd_dir, exist_ok=True) + passwd_file = os.path.join(passwd_dir, "passwd") + group_file = os.path.join(passwd_dir, "group") + + try: + username = pwd.getpwuid(uid).pw_name + except KeyError: + username = "agent" + try: + groupname = grp.getgrgid(gid).gr_name + except KeyError: + groupname = "agent" + + with open(passwd_file, "w") as f: + f.write(f"root:x:0:0:root:/root:/bin/bash\n") + f.write(f"{username}:x:{uid}:{gid}:agent:{container_home}:/bin/bash\n") + with open(group_file, "w") as f: + f.write(f"root:x:0:\n{groupname}:x:{gid}:\n") + + cmd: list[str] = [ + docker, + "run", + "-d", + "--name", + self.container_name, + "--user", + f"{uid}:{gid}", + "--network", + "host", + "-v", + f"{passwd_file}:/etc/passwd:ro", + "-v", + f"{group_file}:/etc/group:ro", + "-e", + f"HOME={container_home}", + "-e", + "GIT_SSH_COMMAND=ssh -o StrictHostKeyChecking=accept-new", + "-e", + f"AGENT_SYS_NO_PERMISSIONS={os.environ.get('AGENT_SYS_NO_PERMISSIONS', '1')}", + "-e", + "AGENT_SYS_REPO=/opt/Infera", + "-e", + f"INFERA_AGENT_SYSTEM_WORKROOT={workroot}", + "-v", + f"{workroot}:{workroot}", + ] + + if self.detect_ssh: + ssh_dir = os.path.join(home, ".ssh") + if os.path.isdir(ssh_dir): + cmd += ["-v", f"{ssh_dir}:{container_home}/.ssh:ro"] + log.info("mounting host SSH config from %s", ssh_dir) + else: + log.info("no ~/.ssh directory found, skipping SSH mount") + + if self.detect_claude: + claude_dir = os.path.join(home, ".claude") + if os.path.isdir(claude_dir): + cmd += ["-v", f"{claude_dir}:{container_home}/.claude"] + log.info("mounting host Claude config from %s", claude_dir) + else: + log.info("no ~/.claude directory found, skipping Claude mount") + + cmd += [self.image, "sleep", "infinity"] + + log.info("starting container %s", self.container_name) + subprocess.run(cmd, check=True) + + # ---------------------------------------------------------------------- # + + def exec(self, argv: list[str], *, workdir: str | None = None) -> int: + """Run a command inside the container and return its exit code.""" + cmd = [self._docker_bin(), "exec"] + if workdir: + cmd += ["-w", workdir] + cmd += [self.container_name, *argv] + result = subprocess.run(cmd) + return result.returncode + + def stop(self) -> None: + """Stop and remove the container.""" + subprocess.run( + [self._docker_bin(), "rm", "-f", self.container_name], + capture_output=True, + ) diff --git a/agent_sys/tests/agent/conftest.py b/agent_sys/tests/agent/conftest.py index f348a1e0f..5ee735ca7 100644 --- a/agent_sys/tests/agent/conftest.py +++ b/agent_sys/tests/agent/conftest.py @@ -51,6 +51,7 @@ def __init__( raise BackendUnsupported(key, "run here", str(self.config["unavailable"])) self.deployed = 0 + self.terminated = 0 self.delivered: list[str] = [] self.interrupted = 0 self.results: list[AgentResult] = list(self.config.get("results") or []) @@ -66,6 +67,9 @@ def _run(self) -> AgentResult: def _deliver(self, message: str) -> None: self.delivered.append(message) + def _terminate(self) -> None: + self.terminated += 1 + def interrupt(self) -> None: self.interrupted += 1 self.status = AgentStatus.INTERRUPTED diff --git a/agent_sys/tests/agent/test_claude_sdk.py b/agent_sys/tests/agent/test_claude_sdk.py index 3e3b14200..f8e859bda 100644 --- a/agent_sys/tests/agent/test_claude_sdk.py +++ b/agent_sys/tests/agent/test_claude_sdk.py @@ -48,6 +48,7 @@ def __init__(self) -> None: # `None` and `query()` raised `AttributeError` on every real run while # this file stayed green. self.connected = False + self.disconnects = 0 self.queries: list[str] = [] self.interrupts = 0 self.responses: list[list[Message]] = [] @@ -57,6 +58,7 @@ async def connect(self) -> None: self.connected = True async def disconnect(self) -> None: + self.disconnects += 1 self.connected = False async def query(self, prompt: str) -> None: @@ -568,6 +570,44 @@ def test_on_started_fires_when_connect_returns() -> None: assert seen == [True] +def test_stop_disconnects_cancels_background_tasks_and_closes_the_loop() -> None: + """A settled submission still owns the SDK reader and subprocess transport. + + `mainloop()` ending is not a client disconnect: the executor deliberately + survives between submissions during one run. Once its owner calls `stop`, + however, no asynchronous task or private event loop may survive it. + """ + + class ClientWithReader(FakeClient): + def __init__(self) -> None: + super().__init__() + self.reader: asyncio.Task | None = None + + async def connect(self) -> None: + await super().connect() + self.reader = asyncio.create_task(self._read_forever()) + + @staticmethod + async def _read_forever() -> None: + await asyncio.Event().wait() + + client = ClientWithReader() + backend = _backend(client) + backend.start() + loop = backend._loop + reader = client.reader + + assert client.connected + assert reader is not None and not reader.done() + + backend.stop() + backend.stop() # terminal cleanup is idempotent + + assert client.disconnects == 1 + assert reader.cancelled() + assert loop.is_closed() + + # ------------------------------------------------- spec §5.5's remote tool surface diff --git a/agent_sys/tests/agent/test_runner.py b/agent_sys/tests/agent/test_runner.py index 6b2930772..e4ea6ed2a 100644 --- a/agent_sys/tests/agent/test_runner.py +++ b/agent_sys/tests/agent/test_runner.py @@ -273,6 +273,30 @@ def test_stop_ends_the_attempt_and_acknowledges(wired) -> None: assert wired.runner.attempt_of(task.id) is None +def test_shutdown_stops_and_forgets_every_surviving_attempt(wired) -> None: + """The CLI owns all attempts after monitors stop and must release them. + + Successful attempts deliberately retain their executor so a monitor can + submit again during the run. Process-level shutdown is the boundary where + that reuse ends, including attempts whose worker thread already returned. + """ + first, _ = _run(wired, "writer", "leaf_ai") + second, _ = _run(wired, "writer", "leaf_ai") + attempts = [wired.runner.attempt_of(task.id) for task in (first, second)] + executors = [attempt.executor for attempt in attempts if attempt is not None] + + assert len(executors) == 2 + assert all(isinstance(executor, ScriptedBackend) for executor in executors) + assert all(executor.terminated == 0 for executor in executors) + + wired.runner.shutdown() + wired.runner.shutdown() # process-level cleanup is idempotent + + assert all(executor.terminated == 1 for executor in executors) + assert wired.runner.attempt_of(first.id) is None + assert wired.runner.attempt_of(second.id) is None + + def test_an_unresolvable_monitor_is_loud(wired) -> None: """`docs/interfaces.md` §2.1 rev. 4: such a task "never advances a phase". It fails visibly instead — a task that hangs for ever and a task that says diff --git a/agent_sys/tests/cli/test_build.py b/agent_sys/tests/cli/test_build.py index b4801c152..e7abaccbc 100644 --- a/agent_sys/tests/cli/test_build.py +++ b/agent_sys/tests/cli/test_build.py @@ -455,6 +455,60 @@ def test_resume_continues_from_disk(tmp_path: Path, package_root: Path) -> None: assert tasks["consume"].history == [] +def test_the_named_work_root_outranks_the_xdg_base(tmp_path: Path, monkeypatch: Any) -> None: + """`INFERA_AGENT_SYSTEM_WORKROOT` names the run root; `XDG_STATE_HOME` a base. + + The two are not two spellings of one setting and the test says which is + which: with both set the specific name wins **whole**, and the loser's value + does not appear in the answer even as a prefix. A first version appended + `agent-sys-demo` to it too, which reads the same in a passing test and puts + the run somewhere the operator did not name. + """ + from cli.environment import WORKROOT_ENV_VAR, default_root + + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "xdg")) + monkeypatch.setenv(WORKROOT_ENV_VAR, str(tmp_path / "work")) + assert default_root() == tmp_path / "work" + + monkeypatch.delenv(WORKROOT_ENV_VAR) + assert default_root() == tmp_path / "xdg" / "agent-sys-demo" + + +@pytest.mark.parametrize("value", ["", " ", "relative/path", "./runs"]) +def test_an_unusable_work_root_reads_as_unset( + value: str, tmp_path: Path, monkeypatch: Any +) -> None: + """Empty or relative falls back rather than being obeyed. + + A relative run root resolves against whatever `cwd` a body inherited, and + `` is the one path that may not depend on that: the bodies reach + their own staged package by absolute path and + `remote.sh:require_visible_on_node` asserts the compute node resolves the + same string. Obeying `./runs` would put the run somewhere that reads + correctly here and is not findable from the other side. + + Its non-vacuity control is the test above, which must still return the + named path — otherwise this one would pass against a function that had + stopped reading the variable at all. + """ + from cli.environment import WORKROOT_ENV_VAR, default_root + + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "xdg")) + monkeypatch.setenv(WORKROOT_ENV_VAR, value) + assert default_root() == tmp_path / "xdg" / "agent-sys-demo" + + +def test_an_explicit_root_outranks_the_environment(tmp_path: Path, monkeypatch: Any) -> None: + """`--demo-root` wins over both. The flag is the most specific statement.""" + from cli.environment import WORKROOT_ENV_VAR, layout_for + + monkeypatch.setenv(WORKROOT_ENV_VAR, str(tmp_path / "work")) + assert layout_for(tmp_path / "explicit").root == tmp_path / "explicit" + # And with no argument the variable is what answers, so the assertion above + # is about precedence rather than about `layout_for` ignoring the environment. + assert layout_for().root == tmp_path / "work" + + def test_two_runs_under_one_root_get_separate_stores(tmp_path: Path) -> None: """**Criterion 13's real case: two runs, one root, back to back.** diff --git a/agent_sys/tests/cli/test_shutdown.py b/agent_sys/tests/cli/test_shutdown.py new file mode 100644 index 000000000..66aa7d308 --- /dev/null +++ b/agent_sys/tests/cli/test_shutdown.py @@ -0,0 +1,69 @@ +"""Process-level ownership of agent executors.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from cli import main as cli_main + + +def _run_harness(monkeypatch: pytest.MonkeyPatch, tmp_path, *, settle_error=None): + runner = SimpleNamespace(shutdown=Mock()) + task_mgr = SimpleNamespace(all=lambda: ()) + registry = SimpleNamespace( + get=lambda name: {"runner": runner, "task_mgr": task_mgr}[name] + ) + monitors = SimpleNamespace(stop=Mock(return_value=[])) + promises = object() + + monkeypatch.setattr(cli_main, "confinement", lambda: "landlock") + monkeypatch.setattr(cli_main, "preflight_credentials", lambda: "ready") + monkeypatch.setattr(cli_main, "preflight_repository", lambda *a, **k: str(tmp_path)) + monkeypatch.setattr(cli_main.package, "locate", lambda path: tmp_path) + monkeypatch.setattr(cli_main.expectations, "for_package", lambda path: promises) + monkeypatch.setattr(cli_main, "_layout", lambda args: SimpleNamespace(run=tmp_path)) + monkeypatch.setattr(cli_main, "permissions_enforced", lambda: True) + monkeypatch.setattr(cli_main, "report_dropped", lambda *a: None) + monkeypatch.setattr(cli_main, "_registry", lambda *a, **k: registry) + monkeypatch.setattr(cli_main, "start_monitors", lambda r: monitors) + monkeypatch.setattr(cli_main, "_start", lambda *a: None) + + def settle(*args, **kwargs): + if settle_error is not None: + raise settle_error + + monkeypatch.setattr(cli_main, "_settle", settle) + monkeypatch.setattr(cli_main, "_emit_graph", lambda *a, **k: None) + monkeypatch.setattr(cli_main, "_describe", lambda *a, **k: None) + monkeypatch.setattr(cli_main, "_report", lambda *a: 0) + + args = SimpleNamespace( + package=str(tmp_path), + allow_repo_config=False, + resume=False, + variables={}, + ) + return args, Mock(), runner, monitors + + +def test_real_run_shuts_down_executors_after_reporting(monkeypatch, tmp_path) -> None: + args, stream, runner, monitors = _run_harness(monkeypatch, tmp_path) + + assert cli_main._real_run(args, stream) == 0 + monitors.stop.assert_called_once_with(timeout=5.0) + runner.shutdown.assert_called_once_with() + + +def test_real_run_shuts_down_executors_when_settle_raises(monkeypatch, tmp_path) -> None: + failure = RuntimeError("settle failed") + args, stream, runner, monitors = _run_harness( + monkeypatch, tmp_path, settle_error=failure + ) + + with pytest.raises(RuntimeError, match="settle failed"): + cli_main._real_run(args, stream) + monitors.stop.assert_called_once_with(timeout=5.0) + runner.shutdown.assert_called_once_with() diff --git a/deploy/docker/Dockerfile.agent-sys b/deploy/docker/Dockerfile.agent-sys new file mode 100644 index 000000000..26d721892 --- /dev/null +++ b/deploy/docker/Dockerfile.agent-sys @@ -0,0 +1,140 @@ +# syntax=docker/dockerfile:1.7 +# +# Control-plane image for agent_sys. +# +# This image carries ONLY agent_sys and its system-layer dependencies. +# Task-specific dependencies (Magpie, compilers, etc.) are declared in each +# task package's environment recipe and installed by env_mgr at runtime. +# +# Secrets (SSH keys, API tokens, .claude) are injected at runtime only. +# +# Build: +# deploy/docker/agent-sys/build.sh +# docker build -f deploy/docker/Dockerfile.agent-sys -t infera/agent-sys:latest . + +ARG PYTHON_VERSION=3.12 +ARG BASE_IMAGE=python:${PYTHON_VERSION}-slim-bookworm + +# =============================================================== stage: base == +# Everything needed at run time, and nothing that only a build needs. +FROM ${BASE_IMAGE} AS base + +ARG NODE_VERSION=v22.23.2 +ARG DOCKER_CLI_VERSION=29.1.3 +ARG TARGETARCH=amd64 + +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_ROOT_USER_ACTION=ignore \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +# bubblewrap is absent: unprivileged containers cannot open user namespaces; +# Landlock works without it. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + rm -f /etc/apt/apt.conf.d/docker-clean \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl gnupg \ + git openssh-client rsync \ + bash coreutils findutils gzip tar xz-utils jq less procps tini \ + && update-ca-certificates + +# uv -- the Python toolchain manager. +COPY --from=ghcr.io/astral-sh/uv:0.9.9 /uv /uvx /usr/local/bin/ + +# `env_mgr/workspace.py` runs `git clone --shared` against /opt/Infera. +# Under `docker run --user 1000:1000` git refuses a repository it does not +# own without safe.directory. +RUN git config --system --add safe.directory '*' \ + && git config --system init.defaultBranch main + +# Node.js + claude CLI. `agent/backends/claude_sdk.py` requires the CLI on +# PATH; it does not fall back to the SDK's bundled executable. +RUN case "${TARGETARCH}" in \ + amd64) NODE_ARCH=x64 ;; \ + arm64) NODE_ARCH=arm64 ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl -fsSL "https://nodejs.org/dist/${NODE_VERSION}/node-${NODE_VERSION}-linux-${NODE_ARCH}.tar.xz" \ + | tar -xJ -C /usr/local --strip-components=1 \ + --exclude='*/CHANGELOG.md' --exclude='*/LICENSE' --exclude='*/README.md' \ + && npm install -g --no-fund --no-audit @anthropic-ai/claude-code \ + && npm cache clean --force \ + && node --version && claude --version + +# Docker client (not daemon) -- for inspecting engine containers over ssh. +ARG WITH_DOCKER_CLI=1 +RUN if [ "${WITH_DOCKER_CLI}" = "1" ]; then \ + case "${TARGETARCH}" in \ + amd64) DOCKER_ARCH=x86_64 ;; \ + arm64) DOCKER_ARCH=aarch64 ;; \ + *) echo "unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl -fsSL "https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_CLI_VERSION}.tgz" \ + | tar -xz -C /usr/local/bin --strip-components=1 docker/docker \ + && docker --version ; \ + fi + +# ========================================================= stage: toolchain == +# Build-only compiler set. Not shipped to the runtime stage. +FROM base AS toolchain +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ + apt-get update \ + && apt-get install -y --no-install-recommends build-essential pkg-config + +# ======================================================== stage: venv-agent == +# agent_sys only. Task-specific packages are installed by env_mgr at runtime. +FROM toolchain AS venv-agent + +COPY . /opt/Infera + +# `--copies` so Landlock confinement works correctly -- see the original +# Dockerfile comments on copies vs symlinks. +RUN python -m venv --copies /opt/venv/agent \ + && /opt/venv/agent/bin/pip install --upgrade pip setuptools wheel + +RUN --mount=type=cache,target=/root/.cache/pip \ + /opt/venv/agent/bin/pip install -e "/opt/Infera/agent_sys[claude,dev]" + +# ============================================================ stage: runtime == +FROM base AS runtime + +LABEL org.opencontainers.image.title="infera-agent-sys" \ + org.opencontainers.image.description="Control plane for agent_sys. Carries no engine and no task-specific dependencies." \ + org.opencontainers.image.source="https://github.com/AMD-AGI/Infera" + +COPY --from=venv-agent /opt/venv/agent /opt/venv/agent +COPY --from=venv-agent /opt/Infera /opt/Infera + +# Git wrapper: restore system config, neutralise user global config. +RUN printf '%s\n' \ + '#!/bin/sh' \ + 'export GIT_CONFIG_SYSTEM=/etc/gitconfig' \ + 'export GIT_CONFIG_GLOBAL=/dev/null' \ + 'unset GIT_CONFIG_NOSYSTEM' \ + 'exec /usr/bin/git "$@"' \ + > /usr/local/bin/git \ + && chmod 0755 /usr/local/bin/git + +ENV VIRTUAL_ENV=/opt/venv/agent \ + PATH=/opt/venv/agent/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/sbin:/usr/sbin:/sbin + +# `prepare()` refuses to clone a repository without preciousObjects. +RUN git -C /opt/Infera config extensions.preciousObjects true + +ENV AGENT_SYS_REPO=/opt/Infera + +# Default run root (container-internal). For real runs, override with -e and +# mount at the identical absolute path on both host and container. +ENV INFERA_AGENT_SYSTEM_WORKROOT=/work/runs + +RUN mkdir -p /work /home/agent \ + && chmod 1777 /work /home/agent + +WORKDIR /opt/Infera + +ENTRYPOINT ["/usr/bin/tini", "--"] +CMD ["bash"] diff --git a/deploy/docker/Dockerfile.agent-sys.dockerignore b/deploy/docker/Dockerfile.agent-sys.dockerignore new file mode 100644 index 000000000..7325866bc --- /dev/null +++ b/deploy/docker/Dockerfile.agent-sys.dockerignore @@ -0,0 +1,34 @@ +# Override root .dockerignore for the agent-sys image. +# .git is required: env_mgr/workspace.py uses git clone --shared against +# /opt/Infera, which reads objects through .git/objects/info/alternates. + +# Python artefacts +__pycache__ +*.py[cod] +*.egg-info +.pytest_cache +.ruff_cache +.mypy_cache +build +dist + +# Virtualenvs +.venv +**/.venv + +# Editors / OS +.vscode +.idea +*.swp +*.log +.DS_Store + +# CI / docs not needed in the image +.github + +# Engine build artefacts +rust/target +deploy/docker/third_party + +# Slurm logs +slurm-*.out diff --git a/deploy/docker/agent-sys/README.md b/deploy/docker/agent-sys/README.md new file mode 100644 index 000000000..d03f0d140 --- /dev/null +++ b/deploy/docker/agent-sys/README.md @@ -0,0 +1,105 @@ +# agent_sys control-plane image + +This image runs `agent_sys` tasks. It contains: + +- `agent_sys[claude,dev]` +- Claude Code CLI and `claude-agent-sdk` +- Python 3.12 in `/opt/venv/agent` +- git, ssh, rsync, uv and the Docker client + +It does **not** contain inference engines, model weights, or task-specific +dependencies (Magpie, compilers, aiperf, etc.). Those are declared in each +task package's environment recipe and installed by `env_mgr` at runtime. + +## Build + +No SSH keys or credentials are needed at build time. + +```bash +deploy/docker/agent-sys/build.sh +deploy/docker/agent-sys/build.sh --tag infera/agent-sys:v1 +``` + +Or directly: + +```bash +docker build -f deploy/docker/Dockerfile.agent-sys -t infera/agent-sys:latest . +``` + +## Usage + +### Standard: start the container, work inside + +```bash +# Start the container +docker run -d --name agent-sys \ + --user "$(id -u):$(id -g)" \ + --network host \ + -e HOME=/home/agent \ + -v "$HOME/.ssh:/home/agent/.ssh:ro" \ + -v "$HOME/.claude:/home/agent/.claude" \ + -e INFERA_AGENT_SYSTEM_WORKROOT="$HOME/.agent_sys_runs" \ + -v "$HOME/.agent_sys_runs:$HOME/.agent_sys_runs" \ + infera/agent-sys:latest \ + sleep infinity + +# Work inside +docker exec -it agent-sys bash + +# Or pass commands through +docker exec agent-sys agent-sys show --package /opt/Infera/agent_sys/examples/demo +``` + +A convenience script is provided at the repository root: + +```bash +./run_agent_sys_container.sh +``` + +### Automated: env_mgr --docker + +If `agent_sys` is installed on the host, `env_mgr` can manage the container +lifecycle automatically: + +```bash +agent-sys run --docker --package /path/to/task +``` + +Options (all default to **on**): + +| Flag | Effect | +|------|--------| +| `--detect-and-copy-host-ssh-config` | Mount host `~/.ssh` into container (read-only) | +| `--detect-and-copy-host-claude-config` | Mount host `~/.claude` into container | +| `--no-detect-and-copy-host-ssh-config` | Skip SSH config detection | +| `--no-detect-and-copy-host-claude-config` | Skip Claude config detection | + +## Container requirements + +- A writable `HOME` (`-e HOME=/home/agent`) +- The caller's UID/GID when a shared filesystem is used +- A run root mounted at the same absolute path on every host that reads it + +Credentials (SSH keys, Claude config) must **not** be stored in the image. +Mount them at runtime. + +## Smoke tests + +```bash +docker exec agent-sys python3 -m pytest -q /opt/Infera/agent_sys + +docker exec agent-sys \ + agent-sys run --dry-run \ + --package /opt/Infera/agent_sys/examples/demo +``` + +## Exit codes + +| Code | Meaning | +| ---: | ------------------------------------------------ | +| `0` | Run completed according to the package contract | +| `1` | Load error | +| `2` | Precondition failure | +| `3` | An expected failure did not occur | +| `4` | A declared expectation was not reached | +| `5` | The task graph did not complete | diff --git a/deploy/docker/agent-sys/build.sh b/deploy/docker/agent-sys/build.sh new file mode 100755 index 000000000..f57b06091 --- /dev/null +++ b/deploy/docker/agent-sys/build.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Build the agent_sys control-plane image. +# +# deploy/docker/agent-sys/build.sh +# deploy/docker/agent-sys/build.sh --tag infera/agent-sys:v1 +# deploy/docker/agent-sys/build.sh --build-arg PYTHON_VERSION=3.13 +# +# No SSH keys or credentials are needed at build time. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HERE/../../.." && pwd)" +DOCKERFILE="$REPO_ROOT/deploy/docker/Dockerfile.agent-sys" + +TAG="" +EXTRA=() + +while [ "$#" -gt 0 ]; do + case "$1" in + --tag|-t) TAG="${2:?}"; shift 2 ;; + -h|--help) sed -n '2,8p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) EXTRA+=("$1"); shift ;; + esac +done + +[ -n "$TAG" ] || TAG="infera/agent-sys:latest" + +echo "[build] tag=$TAG context=$REPO_ROOT" + +exec docker build \ + --file "$DOCKERFILE" \ + --tag "$TAG" \ + --progress=plain \ + "${EXTRA[@]}" \ + "$REPO_ROOT"