diff --git a/src/cli_agent_orchestrator/cli/commands/launch.py b/src/cli_agent_orchestrator/cli/commands/launch.py index 271cd18c3..a0fa50287 100644 --- a/src/cli_agent_orchestrator/cli/commands/launch.py +++ b/src/cli_agent_orchestrator/cli/commands/launch.py @@ -50,6 +50,96 @@ ) _FORWARDED_ENV_MAX_VALUE_BYTES = 2048 +# How long the CLI waits for a freshly created terminal to report ready before +# attaching / sending MESSAGE. Also the FLOOR on the ``POST /sessions`` read +# budget (see ``_create_session_timeout``): the two must not disagree about how +# long provider init is allowed to take, or the create call gives up on work +# the very next step is still willing to wait for. +_READINESS_WAIT_TIMEOUT = 120 + +# Slack added to the create budget on top of the two init-timeout windows. +# Server-side, ``POST /sessions`` also spends time that NEITHER init timeout +# bounds: pane/window creation runs BEFORE ``initialize()`` +# (``terminal_service.create_session``/``create_window``), and inside +# ``initialize()`` there are fixed unbudgeted gaps — e.g. codex's +# ``await asyncio.sleep(2.0)`` shell warm-up, plus send_keys latency and the +# tail of a 1s poll interval on either side of each bounded wait. Without this, +# the sum of those steps could exceed the budget even when the client and +# server agree exactly on the init timeouts (a 60/20 config leaves the client +# at 140s against a worst-case ~141s server path), reopening the same +# silent-drop hole this constant's neighbours exist to close. +_CREATE_OVERHEAD_MARGIN = 30 + + +def _effective_init_timeout(agent_profile, settings): + """Largest ``provider_init_timeout`` the server might apply for this launch. + + Providers do NOT agree on where this value comes from, so the client cannot + assume either source: ``claude_code``/``antigravity_cli``/``kimi_cli`` route + through ``BaseProvider.get_init_timeout(profile)``, which prefers the + profile's own ``provider_init_timeout`` override (that override exists so a + containerized profile whose wrapped CLI is slow can raise its init cap + without touching global config), while ``codex``/``copilot_cli`` read the + global setting directly. Take the MAX of the two: it is the only bound + guaranteed to cover whichever the target provider actually uses, and being + generous here only ever costs an unused ceiling on a request that succeeds + or fails on its own long before it. + + A profile that cannot be loaded (missing / malformed) falls back to the + global value, matching ``get_init_timeout``'s own no-profile behaviour. + """ + global_timeout = int(settings["provider_init_timeout"]) + try: + from cli_agent_orchestrator.utils.agent_profiles import load_agent_profile + + override = load_agent_profile(agent_profile).provider_init_timeout + except Exception: + # Deliberately broad: this is a timeout hint, never a reason to fail a + # launch the server would have accepted. The real profile load happens + # server-side and reports its own errors. + return global_timeout + if override is None: + return global_timeout + return max(global_timeout, int(override)) + + +def _create_session_timeout(settings, agent_profile=None): + """Read budget for ``POST /sessions``, which initializes the provider inline. + + ``POST /sessions`` runs the provider's FULL ``initialize()`` synchronously + server-side (``session_service.create_session`` -> ``create_terminal`` -> + ``await provider_instance.initialize()``), so this request's read budget has + to cover ``provider_init_timeout`` — NOT the generic ``mcp_request_timeout`` + (30s) meant for ordinary tool calls. + + With the 30s budget, a slow provider cold start (codex with MCP servers, on + a container, under a concurrent fan-out) outlived the client: ``requests`` + raised ``ReadTimeout``, ``launch`` turned that into "Failed to connect to + cao-server", and the create-then-send flow below never ran — so MESSAGE was + silently never delivered even though the session, the terminal, and a + healthy idle TUI all existed server-side. Nothing retried, because from the + server's point of view the launch had succeeded. + + Server-side, the init timeout applies TWICE per init — once to + ``wait_for_shell``, again to the final ``wait_until_status`` — with the + startup-prompt/trust-dialog handler in between, so cover the sum rather + than a single init timeout, plus ``_CREATE_OVERHEAD_MARGIN`` for the steps + neither window bounds. The init timeout is resolved per-launch via + ``_effective_init_timeout``, so a profile that raises its own cap widens + this budget with it instead of leaving the client to give up first. + + ``settings.get`` for the handler timeout: a hand-edited partial + settings.json that omits the key must not turn a widened budget into a + ``KeyError`` traceback at launch. + """ + init_timeout = _effective_init_timeout(agent_profile, settings) + return max( + 2 * init_timeout + + int(settings.get("startup_prompt_handler_timeout", 20)) + + _CREATE_OVERHEAD_MARGIN, + _READINESS_WAIT_TIMEOUT, + ) + def _parse_env_pairs(pairs): """Parse repeated ``KEY=VALUE`` entries into a dict, validating each. @@ -297,8 +387,11 @@ def launch( # Forwarded env vars travel in the JSON body so values (which may # contain secrets) don't end up in cao-server's HTTP access log. # See issue #248. - request_timeout = get_server_settings()["mcp_request_timeout"] - post_kwargs: dict = {"params": params, "timeout": request_timeout} + settings = get_server_settings() + post_kwargs: dict = { + "params": params, + "timeout": _create_session_timeout(settings, agents), + } if forwarded_env: post_kwargs["json"] = {"env_vars": forwarded_env} @@ -324,13 +417,14 @@ def launch( ready = wait_until_terminal_status( terminal["id"], {TerminalStatus.IDLE, TerminalStatus.COMPLETED}, - timeout=120, + timeout=_READINESS_WAIT_TIMEOUT, ) if not ready: click.echo( click.style( - f" Warning: {terminal['id']} did not reach idle within 120s — " - "attaching anyway; input may be unreliable until init completes.", + f" Warning: {terminal['id']} did not reach idle within " + f"{_READINESS_WAIT_TIMEOUT}s — attaching anyway; input may be " + "unreliable until init completes.", fg="yellow", ) ) @@ -339,17 +433,17 @@ def launch( ready = wait_until_terminal_status( terminal["id"], {TerminalStatus.IDLE, TerminalStatus.COMPLETED}, - timeout=120, + timeout=_READINESS_WAIT_TIMEOUT, ) if not ready: raise click.ClickException( - f"Conductor {terminal['id']} did not become ready within 120s" + f"Conductor {terminal['id']} did not become ready within " + f"{_READINESS_WAIT_TIMEOUT}s" ) - request_timeout = get_server_settings()["mcp_request_timeout"] response = requests.post( f"{API_BASE_URL}/terminals/{terminal['id']}/input", params={"message": message}, - timeout=request_timeout, + timeout=settings["mcp_request_timeout"], ) response.raise_for_status() time.sleep(3) @@ -357,11 +451,10 @@ def launch( click.echo(f"Message sent to {terminal['name']}. Running in background.") return poll_until_done(terminal["id"], timeout=300) - request_timeout = get_server_settings()["mcp_request_timeout"] output_resp = requests.get( f"{API_BASE_URL}/terminals/{terminal['id']}/output", params={"mode": "last"}, - timeout=request_timeout, + timeout=settings["mcp_request_timeout"], ) output_resp.raise_for_status() output = output_resp.json().get("output", "") diff --git a/src/cli_agent_orchestrator/providers/codex.py b/src/cli_agent_orchestrator/providers/codex.py index 261378cde..55e133ea5 100644 --- a/src/cli_agent_orchestrator/providers/codex.py +++ b/src/cli_agent_orchestrator/providers/codex.py @@ -455,6 +455,16 @@ def _build_codex_command(self) -> str: async def _handle_trust_prompt(self, timeout: float = 20.0) -> None: """Dismiss startup prompts that block readiness. + Every backend call here (get_history/send_keys/send_special_key) is a + blocking subprocess exec, and this loop makes one per second for up to + ``timeout`` seconds. cao-server runs a SINGLE event loop, so leaving + them loop-side froze every other concurrent request — including every + other terminal's own init — for the duration. They are offloaded to + threads for the same reason claude_code's startup handler was in #451; + codex was the slowest remaining provider still doing this, which is why + a concurrent fan-out of codex launches blew the CLI's own read budget on + POST /sessions (see _create_session_timeout in cli/commands/launch.py). + Handles two classes of blocking dialog in a single poll loop: 1. Workspace trust prompt (two variants): @@ -472,7 +482,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None: trust_dismissed = False update_dismissed = False while time.time() - start_time < timeout: - output = get_backend().get_history(self.session_name, self.window_name) + output = await asyncio.to_thread( + get_backend().get_history, self.session_name, self.window_name + ) if not output: await asyncio.sleep(1.0) continue @@ -484,7 +496,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None: logger.info("Codex workspace trust prompt (v1) detected, auto-accepting") status_monitor.notify_input_sent(self.terminal_id) - get_backend().send_special_key(self.session_name, self.window_name, "Enter") + await asyncio.to_thread( + get_backend().send_special_key, self.session_name, self.window_name, "Enter" + ) trust_dismissed = True await asyncio.sleep(1.0) continue @@ -500,7 +514,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None: logger.info("Codex workspace trust prompt (v2) detected, auto-accepting") status_monitor.notify_input_sent(self.terminal_id) - get_backend().send_special_key(self.session_name, self.window_name, "Enter") + await asyncio.to_thread( + get_backend().send_special_key, self.session_name, self.window_name, "Enter" + ) trust_dismissed = True await asyncio.sleep(1.0) continue @@ -512,10 +528,18 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None: "Codex update-available dialog detected, selecting " "'Skip until next version'" ) status_monitor.notify_input_sent(self.terminal_id) - get_backend().send_keys(self.session_name, self.window_name, "3", enter_count=0) + await asyncio.to_thread( + get_backend().send_keys, + self.session_name, + self.window_name, + "3", + enter_count=0, + ) # TUI rendering latency: '3' highlights the menu item, Enter confirms. await asyncio.sleep(0.3) - get_backend().send_special_key(self.session_name, self.window_name, "Enter") + await asyncio.to_thread( + get_backend().send_special_key, self.session_name, self.window_name, "Enter" + ) update_dismissed = True await asyncio.sleep(1.0) continue @@ -540,7 +564,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None: pane_tail = "" try: - output = get_backend().get_history(self.session_name, self.window_name) + output = await asyncio.to_thread( + get_backend().get_history, self.session_name, self.window_name + ) if output: pane_tail = "\n".join(output.splitlines()[-10:]) except Exception: @@ -561,8 +587,10 @@ async def initialize(self) -> bool: # Capture the shell process name before launching codex — used later to # detect when codex has exited and the pane is back to a bare shell. - self.shell_baseline = get_backend().get_pane_current_command( - self.session_name, self.window_name + # Offloaded like the rest of this method's backend calls (#451): each is + # a blocking subprocess exec on cao-server's single shared event loop. + self.shell_baseline = await asyncio.to_thread( + get_backend().get_pane_current_command, self.session_name, self.window_name ) # Send a warm-up command before launching codex. @@ -572,7 +600,9 @@ async def initialize(self) -> bool: # external input that must be allowed to drive PROCESSING transitions # past any previously-latched ready state. status_monitor.notify_input_sent(self.terminal_id) - get_backend().send_keys(self.session_name, self.window_name, "echo ready") + await asyncio.to_thread( + get_backend().send_keys, self.session_name, self.window_name, "echo ready" + ) await asyncio.sleep(2.0) # Build command with flags and agent profile (developer_instructions). @@ -582,10 +612,19 @@ async def initialize(self) -> bool: # caused by the shell_snapshot subprocess inheriting stdin. command = self._build_codex_command() status_monitor.notify_input_sent(self.terminal_id) - get_backend().send_keys(self.session_name, self.window_name, command) + await asyncio.to_thread( + get_backend().send_keys, self.session_name, self.window_name, command + ) - # Handle workspace trust prompt if it appears (new/untrusted directories) - await self._handle_trust_prompt(timeout=20.0) + # Handle workspace trust prompt if it appears (new/untrusted directories). + # Timeout comes from settings so an operator on a slow/containerized host + # can widen it; the hard-coded 20.0 could not be raised without a code + # change, and a cold codex start that renders its first frame later than + # that fell through to "startup prompt handler timed out" and then had to + # be rescued by the wait_until_status below. + await self._handle_trust_prompt( + timeout=float(get_server_settings()["startup_prompt_handler_timeout"]) + ) if not await wait_until_status( self.terminal_id, diff --git a/test/cli/commands/test_launch.py b/test/cli/commands/test_launch.py index 5e742f11a..4bd54aa39 100644 --- a/test/cli/commands/test_launch.py +++ b/test/cli/commands/test_launch.py @@ -6,7 +6,14 @@ import pytest from click.testing import CliRunner -from cli_agent_orchestrator.cli.commands.launch import _parse_env_pairs, launch +from cli_agent_orchestrator.cli.commands.launch import ( + _CREATE_OVERHEAD_MARGIN, + _READINESS_WAIT_TIMEOUT, + _create_session_timeout, + _effective_init_timeout, + _parse_env_pairs, + launch, +) # ── Backend auto-detection (issue #308) ────────────────────────────── @@ -962,3 +969,307 @@ def test_launch_rejects_blocked_env_prefix_before_calling_api(): assert result.exit_code != 0 assert "blocked prefix" in result.output mock_post.assert_not_called() + + +# ── POST /sessions read budget (caom-7it) ───────────────────────────── + + +def test_launch_create_session_timeout_covers_provider_init(): + """``POST /sessions`` must be given a read budget that covers server-side + provider init, not the 30s ``mcp_request_timeout`` meant for tool calls. + + Regression guard for caom-7it: ``POST /sessions`` runs the provider's full + ``initialize()`` inline server-side. Under the old 30s budget a slow codex + cold start outlived the client, ``requests`` raised ``ReadTimeout``, and + ``launch`` reported "Failed to connect to cao-server" — so the + create-then-send flow never ran and MESSAGE was silently never delivered + even though a healthy session and terminal existed. + """ + runner = CliRunner() + + with ( + patch("cli_agent_orchestrator.cli.commands.launch.requests.post") as mock_post, + patch("cli_agent_orchestrator.cli.commands.launch.get_backend"), + patch("cli_agent_orchestrator.cli.commands.launch.wait_until_terminal_status") as mock_wait, + patch("cli_agent_orchestrator.cli.commands.launch.get_server_settings") as mock_settings, + ): + mock_settings.return_value = { + "mcp_request_timeout": 30, + "provider_init_timeout": 60, + "startup_prompt_handler_timeout": 20, + } + mock_post.return_value.json.return_value = { + "session_name": "test-session", + "id": "test-terminal-id", + "name": "test-terminal", + } + mock_post.return_value.raise_for_status.return_value = None + mock_wait.return_value = True + + result = runner.invoke(launch, ["--agents", "test-agent", "--yolo"]) + + assert result.exit_code == 0 + # 2 * provider_init_timeout + startup_prompt_handler_timeout + + # _CREATE_OVERHEAD_MARGIN = 60 + 60 + 20 + 30 = 170. + # The old behaviour passed mcp_request_timeout (30) here. + assert mock_post.call_args.kwargs["timeout"] == 170 + + +def test_launch_create_session_timeout_never_below_readiness_wait(): + """The create budget must not undercut the readiness wait that follows it. + + If ``POST /sessions`` gave up sooner than the CLI's own + ``_READINESS_WAIT_TIMEOUT`` poll is willing to wait, the create call would + abandon work the very next step still expects to complete — the same + silent-drop shape as caom-7it. Small configured init timeouts must floor at + the readiness wait rather than shrink below it. + """ + runner = CliRunner() + + with ( + patch("cli_agent_orchestrator.cli.commands.launch.requests.post") as mock_post, + patch("cli_agent_orchestrator.cli.commands.launch.get_backend"), + patch("cli_agent_orchestrator.cli.commands.launch.wait_until_terminal_status") as mock_wait, + patch("cli_agent_orchestrator.cli.commands.launch.get_server_settings") as mock_settings, + ): + mock_settings.return_value = { + "mcp_request_timeout": 30, + "provider_init_timeout": 5, + "startup_prompt_handler_timeout": 1, + } + mock_post.return_value.json.return_value = { + "session_name": "test-session", + "id": "test-terminal-id", + "name": "test-terminal", + } + mock_post.return_value.raise_for_status.return_value = None + mock_wait.return_value = True + + result = runner.invoke(launch, ["--agents", "test-agent", "--yolo"]) + + assert result.exit_code == 0 + assert mock_post.call_args.kwargs["timeout"] == _READINESS_WAIT_TIMEOUT + + +def test_launch_create_session_timeout_scales_with_configured_init_timeout(): + """An operator who raises ``provider_init_timeout`` for a slow host must get + a proportionally larger create budget — otherwise widening the server-side + allowance has no effect on the client that gives up first.""" + runner = CliRunner() + + with ( + patch("cli_agent_orchestrator.cli.commands.launch.requests.post") as mock_post, + patch("cli_agent_orchestrator.cli.commands.launch.get_backend"), + patch("cli_agent_orchestrator.cli.commands.launch.wait_until_terminal_status") as mock_wait, + patch("cli_agent_orchestrator.cli.commands.launch.get_server_settings") as mock_settings, + ): + mock_settings.return_value = { + "mcp_request_timeout": 30, + "provider_init_timeout": 180, + "startup_prompt_handler_timeout": 45, + } + mock_post.return_value.json.return_value = { + "session_name": "test-session", + "id": "test-terminal-id", + "name": "test-terminal", + } + mock_post.return_value.raise_for_status.return_value = None + mock_wait.return_value = True + + result = runner.invoke(launch, ["--agents", "test-agent", "--yolo"]) + + assert result.exit_code == 0 + # 2 * 180 + 45 + _CREATE_OVERHEAD_MARGIN = 435. + assert mock_post.call_args.kwargs["timeout"] == 435 + + +def test_launch_headless_message_send_still_uses_mcp_request_timeout(): + """Only the create call gets the widened budget. The follow-up ``/input`` + and ``/output`` calls are ordinary requests and must keep using + ``mcp_request_timeout`` — widening those would mask a genuinely hung + server instead of tolerating a slow one-time init.""" + runner = CliRunner() + + with ( + patch("cli_agent_orchestrator.cli.commands.launch.requests.post") as mock_post, + patch("cli_agent_orchestrator.cli.commands.launch.requests.get") as mock_get, + patch("cli_agent_orchestrator.cli.commands.launch.wait_until_terminal_status") as mock_wait, + patch("cli_agent_orchestrator.cli.commands.launch.time.sleep"), + patch("cli_agent_orchestrator.cli.commands.launch.get_server_settings") as mock_settings, + ): + mock_settings.return_value = { + "mcp_request_timeout": 30, + "provider_init_timeout": 60, + "startup_prompt_handler_timeout": 20, + } + mock_post.return_value.json.return_value = { + "session_name": "test-session", + "id": "test-terminal-id", + "name": "test-terminal", + } + mock_post.return_value.raise_for_status.return_value = None + mock_wait.return_value = True + + poll_resp = MagicMock() + poll_resp.raise_for_status.return_value = None + poll_resp.json.return_value = {"status": "completed"} + output_resp = MagicMock() + output_resp.raise_for_status.return_value = None + output_resp.json.return_value = {"output": "task done"} + mock_get.side_effect = [poll_resp, output_resp] + + result = runner.invoke( + launch, + ["--agents", "test-agent", "--headless", "--yolo", "do something"], + ) + + assert result.exit_code == 0 + create_call, input_call = mock_post.call_args_list + assert create_call.kwargs["timeout"] == 170 + assert input_call.kwargs["timeout"] == 30 + assert mock_get.call_args_list[-1].kwargs["timeout"] == 30 + + +# ── Per-profile init-timeout override + overhead margin (caom-7it review) ── + + +def _profile(provider_init_timeout=None): + """Minimal stand-in for a loaded AgentProfile.""" + profile = MagicMock() + profile.provider_init_timeout = provider_init_timeout + return profile + + +@pytest.mark.parametrize( + "override,expected", + [ + # No override: the global governs. + (None, 60), + # Raised override: the profile's larger cap governs (claude_code and + # friends resolve it via BaseProvider.get_init_timeout). + (180, 180), + # LOWERED override: the global still governs, because codex/copilot_cli + # read the global setting directly and would outlive a smaller budget. + (10, 60), + ], +) +def test_effective_init_timeout_takes_max_of_global_and_profile(override, expected): + """The client must bound whichever init timeout the server will actually use. + + Providers disagree on the source: claude_code/antigravity_cli/kimi_cli honour + the per-profile ``provider_init_timeout`` override, while codex/copilot_cli + read the global setting. Only the max of the two covers both, so a raised + override widens the budget and a lowered one cannot shrink it below what a + global-reading provider will still spend. + """ + with patch( + "cli_agent_orchestrator.utils.agent_profiles.load_agent_profile", + return_value=_profile(override), + ): + assert _effective_init_timeout("test-agent", {"provider_init_timeout": 60}) == expected + + +def test_effective_init_timeout_falls_back_when_profile_unloadable(): + """An unloadable profile must fall back to the global, not fail the launch. + + This value is only a timeout hint; the authoritative profile load happens + server-side and reports its own errors. Matches ``get_init_timeout``'s + no-profile behaviour. + """ + with patch( + "cli_agent_orchestrator.utils.agent_profiles.load_agent_profile", + side_effect=FileNotFoundError("no such profile"), + ): + assert _effective_init_timeout("missing-agent", {"provider_init_timeout": 60}) == 60 + + +def test_launch_create_session_timeout_honours_profile_init_timeout_override(): + """A profile that raises ``provider_init_timeout`` must widen the CREATE budget. + + The original caom-7it bug, reintroduced: the profile override exists so a + containerized profile whose wrapped CLI takes far longer can declare a + longer init cap. With the client budgeting only the global 60s, such a + profile runs server-side for up to 2*180+20 while the client gives up at + 170s — ReadTimeout, "Failed to connect to cao-server", MESSAGE dropped + again, for exactly the slow profiles the override exists to serve. + """ + runner = CliRunner() + + with ( + patch("cli_agent_orchestrator.cli.commands.launch.requests.post") as mock_post, + patch("cli_agent_orchestrator.cli.commands.launch.get_backend"), + patch("cli_agent_orchestrator.cli.commands.launch.wait_until_terminal_status") as mock_wait, + patch("cli_agent_orchestrator.cli.commands.launch.get_server_settings") as mock_settings, + patch( + "cli_agent_orchestrator.utils.agent_profiles.load_agent_profile", + return_value=_profile(provider_init_timeout=180), + ), + ): + mock_settings.return_value = { + "mcp_request_timeout": 30, + "provider_init_timeout": 60, + "startup_prompt_handler_timeout": 20, + } + mock_post.return_value.json.return_value = { + "session_name": "test-session", + "id": "test-terminal-id", + "name": "test-terminal", + } + mock_post.return_value.raise_for_status.return_value = None + mock_wait.return_value = True + + result = runner.invoke(launch, ["--agents", "slow-container-agent", "--yolo"]) + + assert result.exit_code == 0 + # 2 * 180 (profile override, not the global 60) + 20 + 30 = 410. + # Budgeting the global would have given 170 and dropped the message. + assert mock_post.call_args.kwargs["timeout"] == 410 + + +def test_launch_create_session_timeout_exceeds_worst_case_server_path(): + """The budget must beat the server's worst case, including unbudgeted steps. + + Neither init-timeout window covers pane creation (which runs BEFORE + ``initialize()``), codex's ``await asyncio.sleep(2.0)`` warm-up, send_keys + latency, or the tail of a 1s poll interval. The reviewer's worked example on + a 60/20 config summed to ~141s server-side against a 140s client budget — + a timeout with no margin at all. Model that path and require the budget to + clear it. + """ + settings = {"provider_init_timeout": 60, "startup_prompt_handler_timeout": 20} + with patch( + "cli_agent_orchestrator.utils.agent_profiles.load_agent_profile", + return_value=_profile(None), + ): + budget = _create_session_timeout(settings, "test-agent") + + create_pane = 3 # create_session/create_window, before initialize() + wait_for_shell = 58 # just under its own cap + warmup = 2 # codex.py's asyncio.sleep(2.0) + send_and_poll = 1 # send_keys + poll-interval tail + trust_prompt = 18 + wait_until_status = 59 + worst_case = ( + create_pane + wait_for_shell + warmup + send_and_poll + trust_prompt + wait_until_status + ) + + assert worst_case == 141, "worked example drifted; re-derive the margin" + assert ( + budget > worst_case + ), f"create budget {budget}s does not cover worst-case server path {worst_case}s" + + +def test_create_session_timeout_survives_partial_settings(): + """A settings.json missing ``startup_prompt_handler_timeout`` must not crash. + + Defense in depth for a hand-edited partial settings file: a KeyError here + would abort the launch before the request is even sent. + """ + with patch( + "cli_agent_orchestrator.utils.agent_profiles.load_agent_profile", + return_value=_profile(None), + ): + budget = _create_session_timeout({"provider_init_timeout": 60}, "test-agent") + + # Falls back to the settings_service default of 20. + assert budget == 2 * 60 + 20 + _CREATE_OVERHEAD_MARGIN diff --git a/test/providers/test_codex_provider_unit.py b/test/providers/test_codex_provider_unit.py index faf9f48ef..6d71899fa 100644 --- a/test/providers/test_codex_provider_unit.py +++ b/test/providers/test_codex_provider_unit.py @@ -2342,3 +2342,169 @@ def test_codex_launch_flags_are_valid(self): assert ( probe.returncode == 0 and "unexpected argument" not in probe.stderr ), f"Flag '{flag}' in launch command rejected by codex binary" + + +class TestCodexInitEventLoopBlocking: + """Codex init must not block cao-server's single shared event loop (caom-7it). + + Every backend call in ``initialize()`` / ``_handle_trust_prompt()`` is a + blocking subprocess exec. cao-server runs ONE event loop, so leaving them + loop-side froze every concurrent request — including every other terminal's + own init — for the duration. Codex was the slowest provider still doing + this: with two codex workers in a fan-out, the self-inflicted queueing + pushed ``POST /sessions`` past the CLI's read budget, the CLI reported + "Failed to connect to cao-server", and the initial MESSAGE was never sent. + + These tests assert the property directly by running an independent heartbeat + task concurrently: a starved loop cannot tick it. + """ + + @staticmethod + async def _heartbeat_gap(coro) -> "tuple[float, int]": + """Run ``coro`` while a 10ms heartbeat runs; return (max gap, tick count). + + The LONGEST interval between two ticks is the metric, not the tick + total: it measures the stall itself, so it is bounded by the size of + the blocking call rather than by the coroutine's total runtime. A total + would also be inflated by the ``await asyncio.sleep()`` gaps in + ``initialize()`` (which correctly yield), making the threshold depend + on scheduler load. The tick count is returned alongside only so a + caller can reject a run where the heartbeat never got to sample. + """ + import asyncio + import time as _t + + ticks = 0 + max_gap = 0.0 + + async def heartbeat() -> None: + nonlocal ticks, max_gap + last = _t.monotonic() + while True: + await asyncio.sleep(0.01) + now = _t.monotonic() + max_gap = max(max_gap, now - last) + last = now + ticks += 1 + + beat = asyncio.create_task(heartbeat()) + try: + # Let the heartbeat actually START before the measured coroutine + # runs. ``create_task`` only schedules it, and the first thing + # ``initialize()`` awaits is a mocked coroutine that completes + # without yielding — so without this the coroutine's synchronous + # backend calls all execute before the heartbeat's first tick and + # the stall goes unobserved (the metric silently passes). + while ticks < 1: + await asyncio.sleep(0.01) + await coro + # And let it tick ONCE MORE afterwards. A gap is only recorded when + # the heartbeat resumes, so a coroutine that blocks and then returns + # without ever suspending (the trust-prompt happy path: one blocking + # get_history, ready frame, return) would be measured as gap-free + # because ``beat.cancel()`` fires first. + final = ticks + while ticks == final: + await asyncio.sleep(0.01) + finally: + beat.cancel() + return max_gap, ticks + + @pytest.mark.asyncio + @patch("cli_agent_orchestrator.providers.codex.get_backend") + async def test_handle_trust_prompt_does_not_starve_the_loop(self, mock_backend): + """A slow ``get_history`` must not freeze other coroutines. + + The backend is stubbed to a genuinely BLOCKING ``time.sleep`` (not an + awaitable) so it stands in for the real subprocess exec. Only a call + offloaded via ``asyncio.to_thread`` lets the heartbeat run. + """ + import time as _time + + def slow_blocking_get_history(*_a, **_kw): + _time.sleep(0.2) + # Ready frame so the handler returns after this single poll. + return "OpenAI Codex (v0.145.0)\n› Explain this codebase\n gpt-5.6-sol high · /tmp\n" + + mock_backend.return_value.get_history.side_effect = slow_blocking_get_history + + provider = CodexProvider("test1234", "test-session", "window-0") + max_gap, ticks = await self._heartbeat_gap(provider._handle_trust_prompt(timeout=20.0)) + + # Offloaded, the worst gap stays near the 10ms heartbeat period; left on + # the loop it is >= the full 0.2s blocking call. 0.1s sits far from both. + assert ticks > 0, "heartbeat never sampled" + assert ( + max_gap < 0.1 + ), f"event loop was starved during _handle_trust_prompt (max gap {max_gap:.3f}s)" + + @pytest.mark.asyncio + @patch("cli_agent_orchestrator.providers.codex.get_server_settings") + @patch("cli_agent_orchestrator.providers.codex.wait_until_status") + @patch("cli_agent_orchestrator.providers.codex.wait_for_shell") + @patch("cli_agent_orchestrator.providers.codex.get_backend") + async def test_initialize_does_not_starve_the_loop( + self, mock_backend, mock_wait_shell, mock_wait_status, mock_settings + ): + """``initialize()``'s own send_keys / get_pane_current_command are + blocking subprocess execs too, not just the trust-prompt poll.""" + import time as _time + + mock_settings.return_value = { + "provider_init_timeout": 60, + "startup_prompt_handler_timeout": 20, + } + mock_wait_shell.return_value = True + mock_wait_status.return_value = True + + def slow_send_keys(*_a, **_kw): + _time.sleep(0.1) + + def slow_get_pane_current_command(*_a, **_kw): + _time.sleep(0.1) + return "zsh" + + mock_backend.return_value.send_keys.side_effect = slow_send_keys + mock_backend.return_value.get_pane_current_command.side_effect = ( + slow_get_pane_current_command + ) + mock_backend.return_value.get_history.return_value = ( + "OpenAI Codex (v0.145.0)\n› Explain this codebase\n gpt-5.6-sol high · /tmp\n" + ) + + provider = CodexProvider("test1234", "test-session", "window-0") + max_gap, ticks = await self._heartbeat_gap(provider.initialize()) + + # Worst-gap, not tick total: initialize()'s own ``await asyncio.sleep`` + # gaps keep ticking the heartbeat, so a total could pass while fully + # blocking. Each stubbed backend call blocks 0.1s, so a loop-side call + # produces a >= 0.1s gap; offloaded, gaps stay near the 10ms period. + assert ticks > 0, "heartbeat never sampled" + assert max_gap < 0.09, f"event loop was starved during initialize (max gap {max_gap:.3f}s)" + + @pytest.mark.asyncio + @patch("cli_agent_orchestrator.providers.codex.get_server_settings") + @patch("cli_agent_orchestrator.providers.codex.wait_until_status") + @patch("cli_agent_orchestrator.providers.codex.wait_for_shell") + @patch("cli_agent_orchestrator.providers.codex.get_backend") + async def test_initialize_uses_configured_startup_prompt_timeout( + self, mock_backend, mock_wait_shell, mock_wait_status, mock_settings + ): + """The trust-prompt budget comes from settings, not a hard-coded 20.0. + + An operator on a slow/containerized host must be able to widen it via + ``startup_prompt_handler_timeout`` without a code change. + """ + mock_settings.return_value = { + "provider_init_timeout": 60, + "startup_prompt_handler_timeout": 45, + } + mock_wait_shell.return_value = True + mock_wait_status.return_value = True + mock_backend.return_value.get_history.return_value = "OpenAI Codex (v0.98.0)" + + provider = CodexProvider("test1234", "test-session", "window-0") + with patch.object(provider, "_handle_trust_prompt", new_callable=AsyncMock) as mock_trust: + await provider.initialize() + + mock_trust.assert_awaited_once_with(timeout=45.0)