Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions src/cli_agent_orchestrator/clients/tmux.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,31 @@ class TmuxClient:
def __init__(self) -> None:
self.server = libtmux.Server()

def _set_server_exit_empty_off(self) -> None:
"""Keep the tmux server alive across a transient zero-session moment.

By default tmux terminates its whole server process the instant the last
session closes (``exit-empty on``). During a mass teardown — many sessions
ending near-simultaneously — that races CAO creating the next session
against the server vanishing: a momentary "no sessions" window tears the
entire server down, taking every other session's panes with it at once
(harness-control#845, the whole-server-death incident). ``exit-empty off``
keeps the server up through an empty moment.

This is the backend belt (HOME-independent, applies to whatever uid runs
cao-server); a HOME ``.tmux.conf`` set is the ops-side complement. Set on
every session-create rather than cached on the client: it is a cheap,
idempotent server option, and setting it each time means it survives even
if the tmux server is ever externally killed and recreated. ``server.cmd``
starts the server if it is not already running, so this also runs before
the very first session exists. Best-effort: a failure here must never block
a session launch.
"""
try:
self.server.cmd("set-option", "-s", "exit-empty", "off")

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Start the tmux server before setting exit-empty

Server.cmd() does not start a server for set-option. With no tmux server running, this command returns error connecting ... and status 1. libtmux 0.51's Server.cmd() returns that result instead of raising, so this except does not log the failure either. new_session() then starts a fresh server with the default exit-empty on.

I reproduced the exact client path on an isolated socket: after the first create_session(), show-options -s exit-empty returned on; after a second create_session(), it returned off. The first call can be the only session creation in a CAO run because workers are added as windows, and the same gap returns after an external server restart, so that lifecycle is still exposed to the teardown/create race this PR is meant to close. Start and configure the server in one tmux command sequence (for example start-server ; set-option ...), inspect the returned status, and add a real no-server regression test.

except Exception:
logger.warning("failed to set tmux server option 'exit-empty off'", exc_info=True)

# ── libtmux listing boundary ─────────────────────────────────────────
#
# Every read that makes libtmux shell out to `list-sessions` /
Expand Down Expand Up @@ -350,6 +375,11 @@ def create_session(
) -> str:
"""Create detached tmux session with initial window and return window name."""
try:
# Ensure the server won't die on a transient empty moment during a
# mass teardown (harness-control#845). Runs before new_session, and
# starts the server if it isn't up yet.
self._set_server_exit_empty_off()

working_directory = self._resolve_and_validate_working_directory(working_directory)

# Only pass essential env vars to avoid tmux "command too long"
Expand Down
27 changes: 27 additions & 0 deletions test/clients/test_tmux_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,33 @@ def test_create_session_success(self, tmux, tmp_path):
assert result == "my-window"
tmux.server.new_session.assert_called_once()

def test_create_session_disables_exit_empty(self, tmux, tmp_path):
"""harness-control#845: creating a session must set the server-wide
'exit-empty off' option (before new_session) so a transient empty moment
during a mass teardown can't take the whole tmux server down."""
mock_window = MagicMock()
mock_window.name = "my-window"
mock_session = MagicMock()
mock_session.windows = [mock_window]
tmux.server.new_session.return_value = mock_session

tmux.create_session("ses", "my-window", "tid1", str(tmp_path))

tmux.server.cmd.assert_any_call("set-option", "-s", "exit-empty", "off")
Comment on lines +75 to +77

def test_exit_empty_failure_does_not_block_launch(self, tmux, tmp_path):
"""Setting exit-empty is best-effort: a failure must NOT abort the launch."""
mock_window = MagicMock()
mock_window.name = "my-window"
mock_session = MagicMock()
mock_session.windows = [mock_window]
tmux.server.new_session.return_value = mock_session
tmux.server.cmd.side_effect = RuntimeError("tmux unavailable")

# Must still succeed despite the set-option failure.
result = tmux.create_session("ses", "my-window", "tid1", str(tmp_path))
assert result == "my-window"

def test_create_session_window_name_none(self, tmux, tmp_path):
mock_window = MagicMock()
mock_window.name = None
Expand Down
Loading